diff options
83 files changed, 1849 insertions, 720 deletions
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 9c021b23db6..ca89c4e2c89 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -28,7 +28,8 @@ package-and-qa-manual:master: extends: .package-and-qa-base only: refs: - - master + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee when: manual needs: ["build-qa-image", "gitlab:assets:compile"] diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 5d93758595a..68f60cb52d4 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -5,7 +5,17 @@ ### Intended users <!-- Who will use this feature? If known, include any of the following: types of users (e.g. Developer), personas, or specific company roles (e.g. Release Manager). It's okay to write "Unknown" and fill this field in later. -Personas can be found at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ --> + +* [Parker (Product Manager)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#parker-product-manager) +* [Delaney (Development Team Lead)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#delaney-development-team-lead) +* [Sasha (Software Developer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sasha-software-developer) +* [Presley (Product Designer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#presley-product-designer) +* [Devon (DevOps Engineer)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#devon-devops-engineer) +* [Sidney (Systems Administrator)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sidney-systems-administrator) +* [Sam (Security Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sam-security-analyst) +* [Dana (Data Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#dana-data-analyst) + +Personas are described at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ --> ### Further details @@ -106,7 +106,7 @@ gem 'fog-aws', '~> 3.5' # Locked until fog-google resolves https://github.com/fog/fog-google/issues/421. # Also see config/initializers/fog_core_patch.rb. gem 'fog-core', '= 2.1.0' -gem 'fog-google', '~> 1.8' +gem 'fog-google', '~> 1.9' gem 'fog-local', '~> 0.6' gem 'fog-openstack', '~> 1.0' gem 'fog-rackspace', '~> 0.1.1' @@ -234,7 +234,7 @@ gem 'asana', '~> 0.8.1' gem 'ruby-fogbugz', '~> 0.2.1' # Kubernetes integration -gem 'kubeclient', '~> 4.2.2' +gem 'kubeclient', '~> 4.4.0' # Sanitize user input gem 'sanitize', '~> 4.6' diff --git a/Gemfile.lock b/Gemfile.lock index 515e1795671..48053e5740e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -284,7 +284,7 @@ GEM excon (~> 0.58) formatador (~> 0.2) mime-types - fog-google (1.8.2) + fog-google (1.9.1) fog-core (<= 2.1.0) fog-json (~> 1.2) fog-xml (~> 0.1.0) @@ -505,7 +505,7 @@ GEM kramdown (2.1.0) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - kubeclient (4.2.2) + kubeclient (4.4.0) http (~> 3.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) @@ -1114,7 +1114,7 @@ DEPENDENCIES fog-aliyun (~> 0.3) fog-aws (~> 3.5) fog-core (= 2.1.0) - fog-google (~> 1.8) + fog-google (~> 1.9) fog-local (~> 0.6) fog-openstack (~> 1.0) fog-rackspace (~> 0.1.1) @@ -1164,7 +1164,7 @@ DEPENDENCIES jwt (~> 2.1.0) kaminari (~> 1.0) knapsack (~> 1.17) - kubeclient (~> 4.2.2) + kubeclient (~> 4.4.0) letter_opener_web (~> 1.3.4) license_finder (~> 5.4) licensee (~> 8.9) diff --git a/app/assets/javascripts/admin/statistics_panel/components/app.vue b/app/assets/javascripts/admin/statistics_panel/components/app.vue new file mode 100644 index 00000000000..29077d926cf --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/components/app.vue @@ -0,0 +1,45 @@ +<script> +import { mapState, mapGetters, mapActions } from 'vuex'; +import { GlLoadingIcon } from '@gitlab/ui'; +import statisticsLabels from '../constants'; + +export default { + components: { + GlLoadingIcon, + }, + data() { + return { + statisticsLabels, + }; + }, + computed: { + ...mapState(['isLoading', 'statistics']), + ...mapGetters(['getStatistics']), + }, + mounted() { + this.fetchStatistics(); + }, + methods: { + ...mapActions(['fetchStatistics']), + }, +}; +</script> + +<template> + <div class="info-well"> + <div class="well-segment admin-well admin-well-statistics"> + <h4>{{ __('Statistics') }}</h4> + <gl-loading-icon v-if="isLoading" size="md" class="my-3" /> + <template v-else> + <p + v-for="statistic in getStatistics(statisticsLabels)" + :key="statistic.key" + class="js-stats" + > + {{ statistic.label }} + <span class="light float-right">{{ statistic.value }}</span> + </p> + </template> + </div> + </div> +</template> diff --git a/app/assets/javascripts/admin/statistics_panel/constants.js b/app/assets/javascripts/admin/statistics_panel/constants.js new file mode 100644 index 00000000000..2dce19a3894 --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/constants.js @@ -0,0 +1,14 @@ +import { s__ } from '~/locale'; + +const statisticsLabels = { + forks: s__('AdminStatistics|Forks'), + issues: s__('AdminStatistics|Issues'), + mergeRequests: s__('AdminStatistics|Merge Requests'), + notes: s__('AdminStatistics|Notes'), + snippets: s__('AdminStatistics|Snippets'), + sshKeys: s__('AdminStatistics|SSH Keys'), + milestones: s__('AdminStatistics|Milestones'), + activeUsers: s__('AdminStatistics|Active Users'), +}; + +export default statisticsLabels; diff --git a/app/assets/javascripts/admin/statistics_panel/index.js b/app/assets/javascripts/admin/statistics_panel/index.js new file mode 100644 index 00000000000..39112e3ddc0 --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/index.js @@ -0,0 +1,22 @@ +import Vue from 'vue'; +import StatisticsPanelApp from './components/app.vue'; +import createStore from './store'; + +export default function(el) { + if (!el) { + return false; + } + + const store = createStore(); + + return new Vue({ + el, + store, + components: { + StatisticsPanelApp, + }, + render(h) { + return h(StatisticsPanelApp); + }, + }); +} diff --git a/app/assets/javascripts/admin/statistics_panel/store/actions.js b/app/assets/javascripts/admin/statistics_panel/store/actions.js new file mode 100644 index 00000000000..537025f524c --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/actions.js @@ -0,0 +1,28 @@ +import Api from '~/api'; +import { s__ } from '~/locale'; +import createFlash from '~/flash'; +import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; +import * as types from './mutation_types'; + +export const requestStatistics = ({ commit }) => commit(types.REQUEST_STATISTICS); + +export const fetchStatistics = ({ dispatch }) => { + dispatch('requestStatistics'); + + Api.adminStatistics() + .then(({ data }) => { + dispatch('receiveStatisticsSuccess', convertObjectPropsToCamelCase(data, { deep: true })); + }) + .catch(error => dispatch('receiveStatisticsError', error)); +}; + +export const receiveStatisticsSuccess = ({ commit }, statistics) => + commit(types.RECEIVE_STATISTICS_SUCCESS, statistics); + +export const receiveStatisticsError = ({ commit }, error) => { + commit(types.RECEIVE_STATISTICS_ERROR, error); + createFlash(s__('AdminDashboard|Error loading the statistics. Please try again')); +}; + +// prevent babel-plugin-rewire from generating an invalid default during karma tests +export default () => {}; diff --git a/app/assets/javascripts/admin/statistics_panel/store/getters.js b/app/assets/javascripts/admin/statistics_panel/store/getters.js new file mode 100644 index 00000000000..24437bc76bf --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/getters.js @@ -0,0 +1,17 @@ +/** + * Merges the statisticsLabels with the state's data + * and returns an array of the following form: + * [{ key: "forks", label: "Forks", value: 50 }] + */ +export const getStatistics = state => labels => + Object.keys(labels).map(key => { + const result = { + key, + label: labels[key], + value: state.statistics && state.statistics[key] ? state.statistics[key] : null, + }; + return result; + }); + +// prevent babel-plugin-rewire from generating an invalid default during karma tests +export default () => {}; diff --git a/app/assets/javascripts/admin/statistics_panel/store/index.js b/app/assets/javascripts/admin/statistics_panel/store/index.js new file mode 100644 index 00000000000..ece9e6419dd --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/index.js @@ -0,0 +1,16 @@ +import Vue from 'vue'; +import Vuex from 'vuex'; +import * as actions from './actions'; +import * as getters from './getters'; +import mutations from './mutations'; +import state from './state'; + +Vue.use(Vuex); + +export default () => + new Vuex.Store({ + actions, + getters, + mutations, + state: state(), + }); diff --git a/app/assets/javascripts/admin/statistics_panel/store/mutation_types.js b/app/assets/javascripts/admin/statistics_panel/store/mutation_types.js new file mode 100644 index 00000000000..4e0ca4ed3cd --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/mutation_types.js @@ -0,0 +1,3 @@ +export const REQUEST_STATISTICS = 'REQUEST_STATISTICS'; +export const RECEIVE_STATISTICS_SUCCESS = 'RECEIVE_STATISTICS_SUCCESS'; +export const RECEIVE_STATISTICS_ERROR = 'RECEIVE_STATISTICS_ERROR'; diff --git a/app/assets/javascripts/admin/statistics_panel/store/mutations.js b/app/assets/javascripts/admin/statistics_panel/store/mutations.js new file mode 100644 index 00000000000..d0fac5cfbab --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/mutations.js @@ -0,0 +1,16 @@ +import * as types from './mutation_types'; + +export default { + [types.REQUEST_STATISTICS](state) { + state.isLoading = true; + }, + [types.RECEIVE_STATISTICS_SUCCESS](state, data) { + state.isLoading = false; + state.error = null; + state.statistics = data; + }, + [types.RECEIVE_STATISTICS_ERROR](state, error) { + state.isLoading = false; + state.error = error; + }, +}; diff --git a/app/assets/javascripts/admin/statistics_panel/store/state.js b/app/assets/javascripts/admin/statistics_panel/store/state.js new file mode 100644 index 00000000000..f2f2dc0a4d2 --- /dev/null +++ b/app/assets/javascripts/admin/statistics_panel/store/state.js @@ -0,0 +1,5 @@ +export default () => ({ + error: null, + isLoading: false, + statistics: null, +}); diff --git a/app/assets/javascripts/api.js b/app/assets/javascripts/api.js index 136ffdf8b9d..1d97ad5ec11 100644 --- a/app/assets/javascripts/api.js +++ b/app/assets/javascripts/api.js @@ -36,6 +36,7 @@ const Api = { branchSinglePath: '/api/:version/projects/:id/repository/branches/:branch', createBranchPath: '/api/:version/projects/:id/repository/branches', releasesPath: '/api/:version/projects/:id/releases', + adminStatisticsPath: 'api/:version/application/statistics', group(groupId, callback) { const url = Api.buildUrl(Api.groupPath).replace(':id', groupId); @@ -376,6 +377,11 @@ const Api = { return axios.get(url); }, + adminStatistics() { + const url = Api.buildUrl(this.adminStatisticsPath); + return axios.get(url); + }, + buildUrl(url) { return joinPaths(gon.relative_url_root || '', url.replace(':version', gon.api_version)); }, diff --git a/app/assets/javascripts/pages/admin/index.js b/app/assets/javascripts/pages/admin/index.js index 8a32556f06c..74f2eead755 100644 --- a/app/assets/javascripts/pages/admin/index.js +++ b/app/assets/javascripts/pages/admin/index.js @@ -1,3 +1,8 @@ import initAdmin from './admin'; +import initAdminStatisticsPanel from '../../admin/statistics_panel/index'; -document.addEventListener('DOMContentLoaded', initAdmin()); +document.addEventListener('DOMContentLoaded', () => { + const statisticsPanelContainer = document.getElementById('js-admin-statistics-container'); + initAdmin(); + initAdminStatisticsPanel(statisticsPanelContainer); +}); diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index 23cc9ee247a..64b959e2431 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -3,8 +3,7 @@ class Admin::DashboardController < Admin::ApplicationController include CountHelper - COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, - MergeRequest, Note, Snippet, Key, Milestone].freeze + COUNTED_ITEMS = [Project, User, Group].freeze # rubocop: disable CodeReuse/ActiveRecord def index diff --git a/app/helpers/user_callouts_helper.rb b/app/helpers/user_callouts_helper.rb index d5e459311f7..f10fadfdf49 100644 --- a/app/helpers/user_callouts_helper.rb +++ b/app/helpers/user_callouts_helper.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true module UserCalloutsHelper - GKE_CLUSTER_INTEGRATION = 'gke_cluster_integration'.freeze - GCP_SIGNUP_OFFER = 'gcp_signup_offer'.freeze - SUGGEST_POPOVER_DISMISSED = 'suggest_popover_dismissed'.freeze + GKE_CLUSTER_INTEGRATION = 'gke_cluster_integration' + GCP_SIGNUP_OFFER = 'gcp_signup_offer' + SUGGEST_POPOVER_DISMISSED = 'suggest_popover_dismissed' def show_gke_cluster_integration_callout?(project) can?(current_user, :create_cluster, project) && diff --git a/app/models/issue.rb b/app/models/issue.rb index 75d4fc8c1c5..7c5a139ab55 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -31,7 +31,7 @@ class Issue < ApplicationRecord has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.issues&.maximum(:iid) } - has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 95daa48d4bc..901ebcf249f 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -54,7 +54,7 @@ class MergeRequest < ApplicationRecord belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" - has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 915978d37b8..7f46e5faf1a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -37,7 +37,7 @@ class Milestone < ApplicationRecord has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests - has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent scope :of_projects, ->(ids) { where(project_id: ids) } scope :of_groups, ->(ids) { where(group_id: ids) } diff --git a/app/models/note.rb b/app/models/note.rb index 0d024b0a25c..5bd3a7f969a 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -78,7 +78,7 @@ class Note < ApplicationRecord # suggestions.delete_all calls has_many :suggestions, -> { order(:relative_order) }, inverse_of: :note, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata has_one :note_diff_file, inverse_of: :diff_note, foreign_key: :diff_note_id diff --git a/app/models/user.rb b/app/models/user.rb index 67d730e2fa3..5f109feb96a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -131,7 +131,7 @@ class User < ApplicationRecord has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :issues, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :merge_requests, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent - has_many :events, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :events, dependent: :delete_all, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :releases, dependent: :nullify, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :subscriptions, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 8aca61efe7b..8fad42436ca 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -35,41 +35,7 @@ = link_to 'New group', new_admin_group_path, class: "btn btn-success" .row .col-md-4 - .info-well - .well-segment.admin-well.admin-well-statistics - %h4 Statistics - %p - Forks - %span.light.float-right - = approximate_fork_count_with_delimiters(@counts) - %p - Issues - %span.light.float-right - = approximate_count_with_delimiters(@counts, Issue) - %p - Merge Requests - %span.light.float-right - = approximate_count_with_delimiters(@counts, MergeRequest) - %p - Notes - %span.light.float-right - = approximate_count_with_delimiters(@counts, Note) - %p - Snippets - %span.light.float-right - = approximate_count_with_delimiters(@counts, Snippet) - %p - SSH Keys - %span.light.float-right - = approximate_count_with_delimiters(@counts, Key) - %p - Milestones - %span.light.float-right - = approximate_count_with_delimiters(@counts, Milestone) - %p - Active Users - %span.light.float-right - = number_with_delimiter(User.active.count) + #js-admin-statistics-container .col-md-4 .info-well .well-segment.admin-well.admin-well-features diff --git a/changelogs/unreleased/46686-bump-kubeclient-version-qa.yml b/changelogs/unreleased/46686-bump-kubeclient-version-qa.yml new file mode 100644 index 00000000000..b4adbe4dcea --- /dev/null +++ b/changelogs/unreleased/46686-bump-kubeclient-version-qa.yml @@ -0,0 +1,5 @@ +--- +title: Bump Kubeclient to 4.4.0 +merge_request: 32811 +author: +type: other diff --git a/changelogs/unreleased/51123-error-500-viewing-admin-page-due-to-statement-timeout-on-counting-t.yml b/changelogs/unreleased/51123-error-500-viewing-admin-page-due-to-statement-timeout-on-counting-t.yml new file mode 100644 index 00000000000..9862137c80c --- /dev/null +++ b/changelogs/unreleased/51123-error-500-viewing-admin-page-due-to-statement-timeout-on-counting-t.yml @@ -0,0 +1,5 @@ +--- +title: 'Admin dashboard: Fetch and render statistics async' +merge_request: 32449 +author: +type: other diff --git a/changelogs/unreleased/job-rules-e2e.yml b/changelogs/unreleased/job-rules-e2e.yml new file mode 100644 index 00000000000..2c55dfcec49 --- /dev/null +++ b/changelogs/unreleased/job-rules-e2e.yml @@ -0,0 +1,5 @@ +--- +title: Passing job rules downstream and E2E specs for job:rules configuration +merge_request: 32609 +author: +type: fixed diff --git a/db/migrate/20190806071559_remove_epic_issues_default_relative_position.rb b/db/migrate/20190806071559_remove_epic_issues_default_relative_position.rb index f6db90f6637..3037f2ea106 100644 --- a/db/migrate/20190806071559_remove_epic_issues_default_relative_position.rb +++ b/db/migrate/20190806071559_remove_epic_issues_default_relative_position.rb @@ -3,8 +3,23 @@ class RemoveEpicIssuesDefaultRelativePosition < ActiveRecord::Migration[5.2] DOWNTIME = false - def change - change_column_null :epic_issues, :relative_position, true - change_column_default :epic_issues, :relative_position, from: 1073741823, to: nil + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + # The column won't exist if someone installed EE, downgraded to CE + # before it was added in EE, then tries to upgrade CE. + if column_exists?(:epic_issues, :relative_position) + change_column_null :epic_issues, :relative_position, true + change_column_default :epic_issues, :relative_position, from: 1073741823, to: nil + else + add_column_with_default(:epic_issues, :relative_position, :integer, default: nil, allow_null: true) + end + end + + def down + change_column_default :epic_issues, :relative_position, from: nil, to: 1073741823 + change_column_null :epic_issues, :relative_position, false end end diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index df6c554decb..318711fd281 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -54,6 +54,21 @@ http://localhost:8080/plantuml you can change these defaults by editing the `/etc/tomcat7/server.xml` file. +### Making local PlantUML accessible using custom GitLab setup + +The PlantUML server runs locally on your server, so it is not accessible +externally. As such, it is necessary to catch external PlantUML calls and +redirect them to the local server. + +The idea is to redirect each call to `https://gitlab.example.com/-/plantuml/` +to the local PlantUML server `http://localhost:8080/plantuml`. + +To enable the redirection, add the following line in `/etc/gitlab/gitlab.rb`: + +```ruby +nginx['custom_gitlab_server_config'] = "location /-/plantuml { \n proxy_cache off; \n proxy_pass http://127.0.0.1:8080; \n}\n" +``` + ## GitLab You need to enable PlantUML integration from Settings under Admin Area. To do @@ -62,7 +77,7 @@ that, login with an Admin account and do following: - In GitLab, go to **Admin Area > Settings > Integrations**. - Expand the **PlantUML** section. - Check **Enable PlantUML** checkbox. -- Set the PlantUML instance as **PlantUML URL**. +- Set the PlantUML instance as `https://gitlab.example.com/-/plantuml/`. ## Creating Diagrams diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index 9af5430f1c8..e2ddc2cbc18 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -126,6 +126,7 @@ The following API resources are available outside of project and group contexts | [Runners](runners.md) | `/runners` (also available for projects) | | [Search](search.md) | `/search` (also available for groups and projects) | | [Settings](settings.md) | `/application/settings` | +| [Statistics](statistics.md) | `/application/statistics` | | [Sidekiq metrics](sidekiq_metrics.md) | `/sidekiq` | | [Suggestions](suggestions.md) | `/suggestions` | | [System hooks](system_hooks.md) | `/hooks` | diff --git a/doc/api/statistics.md b/doc/api/statistics.md new file mode 100644 index 00000000000..5078b2f26d4 --- /dev/null +++ b/doc/api/statistics.md @@ -0,0 +1,35 @@ +# Application statistics API + +## Get current application statistics + +List the current statistics of the GitLab instance. You have to be an +administrator in order to perform this action. + +NOTE: **Note:** +These statistics are approximate. + +``` +GET /application/statistics +``` + +```bash +curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/application/statistics +``` + +Example response: + +```json +{ + "forks": "10", + "issues": "76", + "merge_requests": "27", + "notes": "954", + "snippets": "50", + "ssh_keys": "10", + "milestones": "40", + "users": "50", + "groups": "10", + "projects": "20", + "active_users": "50" +} +``` diff --git a/doc/api/v3_to_v4.md b/doc/api/v3_to_v4.md index 5f875528a6c..b6059c71b27 100644 --- a/doc/api/v3_to_v4.md +++ b/doc/api/v3_to_v4.md @@ -23,8 +23,8 @@ Below are the changes made between V3 and V4. - Status 409 returned for `POST /projects/:id/members` when a member already exists [!9093](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9093) - Moved `DELETE /projects/:id/star` to `POST /projects/:id/unstar` [!9328](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9328) - Removed the following deprecated Templates endpoints (these are still accessible with `/templates` prefix) [!8853](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8853) - - `/licences` - - `/licences/:key` + - `/licenses` + - `/licenses/:key` - `/gitignores` - `/gitlab_ci_ymls` - `/dockerfiles` diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index a48da557e09..fc0125fcc18 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -721,6 +721,13 @@ deploy: - master ``` +NOTE: **Note:** +This example explicitly calls `docker pull`. If you prefer to implicitly pull the +built image using `image:`, and use either the [Docker](https://docs.gitlab.com/runner/executors/docker.html) +or [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executor, +make sure that [`pull_policy`](https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work) +is set to `always`. + [docker-in-docker]: https://blog.docker.com/2013/09/docker-can-now-run-within-docker/ [docker-cap]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities [2fa]: ../../user/profile/account/two_factor_authentication.md diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 8f2e95dbb10..8aae0e85c89 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2086,7 +2086,9 @@ staging: ### `interruptible` -`interruptible` is used to indicate that a job should be canceled if made redundant by a newer run of the same job. Defaults to `false` if there is an environment defined and `true` otherwise. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23464) in GitLab 12.3. + +`interruptible` is used to indicate that a job should be canceled if made redundant by a newer run of the same job. Defaults to `true`. This value will only be used if the [automatic cancellation of redundant pipelines feature](https://docs.gitlab.com/ee/user/project/pipelines/settings.html#auto-cancel-pending-pipelines) is enabled. diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index a456bbc781f..606ee431c3e 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -302,18 +302,18 @@ for more details. ## Best practices -1. Make sure to know how much data you're dealing with +1. Make sure to know how much data you're dealing with. 1. Make sure that background migration jobs are idempotent. 1. Make sure that tests you write are not false positives. 1. Make sure that if the data being migrated is critical and cannot be lost, the clean-up migration also checks the final state of the data before completing. -1. Make sure to know how much time it'll take to run all scheduled migrations +1. Make sure to know how much time it'll take to run all scheduled migrations. 1. When migrating many columns, make sure it won't generate too many dead tuples in the process (you may need to directly query the number of dead tuples - and adjust the scheduling according to this piece of data) + and adjust the scheduling according to this piece of data). 1. Make sure to discuss the numbers with a database specialist, the migration may add more pressure on DB than you expect (measure on staging, - or ask someone to measure on production) + or ask someone to measure on production). [migrations-readme]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/migrations/README.md [issue-rspec-hooks]: https://gitlab.com/gitlab-org/gitlab-ce/issues/35351 diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index 5fc5886e3a2..6bf59209d21 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -77,7 +77,7 @@ complex logic related to that task. Danger code is just Ruby code. It should adhere to our coding standards, and needs tests, like any other piece of Ruby in our codebase. However, we aren't -able to test a `Dangerfile` directly! So, to maximise test coverage, try to +able to test a `Dangerfile` directly! So, to maximize test coverage, try to minimize the number of lines of code in `danger/`. A non-trivial `Dangerfile` should mostly call plugin code with arguments derived from the methods provided by Danger. The plugin code itself should have unit tests. diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 09dd31e2aee..4a50e90a26c 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -85,7 +85,7 @@ The more we reflexively add useful information to the docs, the more (and more s If you have questions when considering, authoring, or editing docs, ask the Technical Writing team on Slack in `#docs` or in GitLab by mentioning the writer for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). Otherwise, forge ahead with your best effort. It does not need to be perfect; the team is happy to review and improve upon your content. Please review the [Documentation guidelines](index.md) before you begin your first documentation MR. -Having a knowledge base is any form that is separate from the documentation would be against the docs-first methodology because the content would overlap with the documentation. +Having a knowledge base in any form that is separate from the documentation would be against the docs-first methodology because the content would overlap with the documentation. ## Markdown diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index ef319f7f0ce..f1456146032 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -298,3 +298,33 @@ gitlab_rails['omniauth_allow_bypass_two_factor'] = ['twitter', 'google_oauth2'] omniauth: allow_bypass_two_factor: ['twitter', 'google_oauth2'] ``` + +## Automatically sign in with provider + +You can add the `auto_sign_in_with_provider` setting to your +GitLab configuration to automatically redirect login requests +to your OmniAuth provider for authentication, thus removing the need to click a button +before actually signing in. + +For example, when using the Azure integration, you would set the following +to enable auto sign in. + +For Omnibus package: + +```ruby +gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'azure_oauth2' +``` + +For installations from source: + +```yaml +omniauth: + auto_sign_in_with_provider: azure_oauth2 +``` + +Please keep in mind that every sign in attempt will be redirected to the OmniAuth provider, +so you will not be able to sign in using local credentials. Make sure that at least one +of the OmniAuth users has admin permissions. + +You may also bypass the auto signin feature by browsing to +`https://gitlab.example.com/users/sign_in?auto_sign_in=false`. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 59b775d13bd..06048ad618f 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -190,7 +190,7 @@ it` may occur, and will cause the backup process to fail. To combat this, 8.17 introduces a new backup strategy called `copy`. The strategy copies data files to a temporary location before calling `tar` and `gzip`, avoiding the error. -A side-effect is that the backup process with take up to an additional 1X disk +A side-effect is that the backup process will take up to an additional 1X disk space. The process does its best to clean up the temporary files at each stage so the problem doesn't compound, but it could be a considerable change for large installations. This is why the `copy` strategy is not the default in 8.17. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index fa2df667031..89526c08e7e 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -46,6 +46,10 @@ To run a Dependency Scanning job, you need GitLab Runner with the executor running in privileged mode. If you're using the shared Runners on GitLab.com, this is enabled by default. +CAUTION: **Caution:** +If you use your own Runners, make sure that the Docker version you have installed +is **not** `19.03.00`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. + ## Supported languages and package managers The following languages and dependency managers are supported. @@ -343,14 +347,11 @@ You can search the [gemnasium-db](https://gitlab.com/gitlab-org/security-product to find a vulnerability in the Gemnasium database. You can also [submit new vulnerabilities](https://gitlab.com/gitlab-org/security-products/gemnasium-db/blob/master/CONTRIBUTING.md). -<!-- ## Troubleshooting +## Troubleshooting -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +### Error response from daemon: error processing tar file: docker-tar: relocation error -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This error occurs when the Docker version used to run the SAST job is `19.03.00`. +You are advised to update to Docker `19.03.01` or greater. Older versions are not +affected. Read more in +[this issue](https://gitlab.com/gitlab-org/gitlab-ee/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 15a21bb82e0..956d3ef7c8c 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -51,6 +51,10 @@ To run a SAST job, you need GitLab Runner with the executor running in privileged mode. If you're using the shared Runners on GitLab.com, this is enabled by default. +CAUTION: **Caution:** +If you use your own Runners, make sure that the Docker version you have installed +is **not** `19.03.00`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details. + ## Supported languages and frameworks The following table shows which languages, package managers and frameworks are supported and which tools are used. @@ -350,14 +354,11 @@ Once a vulnerability is found, you can interact with it. Read more on how to For more information about the vulnerabilities database update, check the [maintenance table](../index.md#maintenance-and-update-of-the-vulnerabilities-database). -<!-- ## Troubleshooting +## Troubleshooting -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +### Error response from daemon: error processing tar file: docker-tar: relocation error -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +This error occurs when the Docker version used to run the SAST job is `19.03.00`. +You are advised to update to Docker `19.03.01` or greater. Older versions are not +affected. Read more in +[this issue](https://gitlab.com/gitlab-org/gitlab-ee/issues/13830#note_211354992 "Current SAST container fails"). diff --git a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png Binary files differindex 1fe76a9e08f..1fe76a9e08f 100755..100644 --- a/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png +++ b/doc/user/application_security/security_dashboard/img/group_security_dashboard_v12_3.png diff --git a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png Binary files differindex 09979ba99b3..09979ba99b3 100755..100644 --- a/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png +++ b/doc/user/application_security/security_dashboard/img/pipeline_security_dashboard_v12_3.png diff --git a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png Binary files differindex 51e80bdb50d..51e80bdb50d 100755..100644 --- a/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png +++ b/doc/user/application_security/security_dashboard/img/project_security_dashboard_v12_3.png diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index f922b8911ae..65b654d1553 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -129,7 +129,8 @@ file. ### JupyterHub -> Available for project-level clusters since GitLab 11.0. +> - Available for project-level clusters since GitLab 11.0. +> - Available for group-level clusters since GitLab 12.3. [JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) is a multi-user service for managing notebooks across a team. [Jupyter @@ -138,8 +139,9 @@ web-based interactive programming environment used for data analysis, visualization, and machine learning. Authentication will be enabled only for [project -members](../project/members/index.md) with [Developer or -higher](../permissions.md) access to the project. +members](../project/members/index.md) for project-level clusters and group +members for group-level clusters with [Developer or +higher](../permissions.md) access to the associated project or group. We use a [custom Jupyter image](https://gitlab.com/gitlab-org/jupyterhub-user-image/blob/master/Dockerfile) diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 9bf400e7dff..ecd8f74194e 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -194,7 +194,7 @@ terminal: Once the terminal has started, the console will be displayed and we could access the project repository files. -**Important**. The terminal job is branch dependant. This means that the +**Important**. The terminal job is branch dependent. This means that the configuration file used to trigger and configure the terminal will be the one in the selected branch of the Web IDE. diff --git a/lib/api/api.rb b/lib/api/api.rb index aa6a67d817a..88d411e22a9 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -161,6 +161,7 @@ module API mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::Snippets + mount ::API::Statistics mount ::API::Submodules mount ::API::Subscriptions mount ::API::Suggestions diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f7cd6d35854..c9b3483acaf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1169,6 +1169,55 @@ module API expose :message, :starts_at, :ends_at, :color, :font end + class ApplicationStatistics < Grape::Entity + include ActionView::Helpers::NumberHelper + include CountHelper + + expose :forks do |counts| + approximate_fork_count_with_delimiters(counts) + end + + expose :issues do |counts| + approximate_count_with_delimiters(counts, ::Issue) + end + + expose :merge_requests do |counts| + approximate_count_with_delimiters(counts, ::MergeRequest) + end + + expose :notes do |counts| + approximate_count_with_delimiters(counts, ::Note) + end + + expose :snippets do |counts| + approximate_count_with_delimiters(counts, ::Snippet) + end + + expose :ssh_keys do |counts| + approximate_count_with_delimiters(counts, ::Key) + end + + expose :milestones do |counts| + approximate_count_with_delimiters(counts, ::Milestone) + end + + expose :users do |counts| + approximate_count_with_delimiters(counts, ::User) + end + + expose :projects do |counts| + approximate_count_with_delimiters(counts, ::Project) + end + + expose :groups do |counts| + approximate_count_with_delimiters(counts, ::Group) + end + + expose :active_users do |_| + number_with_delimiter(::User.active.count) + end + end + class ApplicationSetting < Grape::Entity def self.exposed_attributes attributes = ::ApplicationSettingsHelper.visible_attributes diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb new file mode 100644 index 00000000000..d2dce34dfa5 --- /dev/null +++ b/lib/api/statistics.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + class Statistics < Grape::API + before { authenticated_as_admin! } + + COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, + MergeRequest, Note, Snippet, Key, Milestone].freeze + + desc 'Get the current application statistics' do + success Entities::ApplicationStatistics + end + get "application/statistics" do + counts = Gitlab::Database::Count.approximate_counts(COUNTED_ITEMS) + present counts, with: Entities::ApplicationStatistics + end + end +end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 89623a809c9..43399c74457 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,7 +6,14 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in) + Result = Struct.new(:when, :start_in) do + def build_attributes + { + when: self.when, + options: { start_in: start_in }.compact + }.compact + end + end def initialize(rule_hashes, default_when = 'on_success') @rule_list = Rule.fabricate_list(rule_hashes) diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb index ff0baf3348c..bf787fe95a6 100644 --- a/lib/gitlab/ci/build/rules/rule/clause.rb +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -13,9 +13,7 @@ module Gitlab UnknownClauseError = Class.new(StandardError) def self.fabricate(type, value) - type = type.to_s.camelize - - self.const_get(type).new(value) if self.const_defined?(type) + "#{self}::#{type.to_s.camelize}".safe_constantize&.new(value) end def initialize(spec) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 3009c7e8329..f750886a8c5 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -122,7 +122,7 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :environment, :coverage, :retry, + :artifacts, :environment, :coverage, :retry, :rules, :parallel, :needs, :interruptible attributes :script, :tags, :allow_failure, :when, :dependencies, @@ -145,6 +145,13 @@ module Gitlab end @entries.delete(:type) + + # This is something of a hack, see issue for details: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/67150 + if !only_defined? && has_rules? + @entries.delete(:only) + @entries.delete(:except) + end end inherit!(deps) @@ -203,6 +210,7 @@ module Gitlab cache: cache_value, only: only_value, except: except_value, + rules: has_rules? ? rules_value : nil, variables: variables_defined? ? variables_value : {}, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 65cad0880f5..2fbc3d9e367 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -26,6 +26,10 @@ module Gitlab end end end + + def value + @config + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 1066331062b..1f6b3853069 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -145,7 +145,7 @@ module Gitlab def rules_attributes strong_memoize(:rules_attributes) do - @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {} + @using_rules ? @rules.evaluate(@pipeline, self).build_attributes : {} end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 501d91fa9ad..986605efdc3 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -42,6 +42,7 @@ module Gitlab yaml_variables: yaml_variables(name), needs_attributes: job[:needs]&.map { |need| { name: need } }, interruptible: job[:interruptible], + rules: job[:rules], options: { image: job[:image], services: job[:services], diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 42cd94add79..13883ca7f3d 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,35 +3,19 @@ module Gitlab module ImportExport class AttributesFinder - def initialize(included_attributes:, excluded_attributes:, methods:) - @included_attributes = included_attributes || {} - @excluded_attributes = excluded_attributes || {} - @methods = methods || {} + def initialize(config:) + @tree = config[:tree] || {} + @included_attributes = config[:included_attributes] || {} + @excluded_attributes = config[:excluded_attributes] || {} + @methods = config[:methods] || {} end - def find(model_object) - parsed_hash = find_attributes_only(model_object) - parsed_hash.empty? ? model_object : { model_object => parsed_hash } + def find_root(model_key) + find(model_key, @tree[model_key]) end - def parse(model_object) - parsed_hash = find_attributes_only(model_object) - yield parsed_hash unless parsed_hash.empty? - end - - def find_included(value) - key = key_from_hash(value) - @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } - end - - def find_excluded(value) - key = key_from_hash(value) - @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } - end - - def find_method(value) - key = key_from_hash(value) - @methods[key].nil? ? {} : { methods: @methods[key] } + def find_relations_tree(model_key) + @tree[model_key] end def find_excluded_keys(klass_name) @@ -40,12 +24,24 @@ module Gitlab private - def find_attributes_only(value) - find_included(value).merge(find_excluded(value)).merge(find_method(value)) + def find(model_key, model_tree) + { + only: @included_attributes[model_key], + except: @excluded_attributes[model_key], + methods: @methods[model_key], + include: resolve_model_tree(model_tree) + }.compact + end + + def resolve_model_tree(model_tree) + return unless model_tree + + model_tree + .map(&method(:resolve_model)) end - def key_from_hash(value) - value.is_a?(Hash) ? value.first.first : value + def resolve_model(model_key, model_tree) + { model_key => find(model_key, model_tree) } end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index f6cd4eb5e0c..6f4919ead4e 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -3,70 +3,49 @@ module Gitlab module ImportExport class Config + def initialize + @hash = parse_yaml + @hash.deep_symbolize_keys! + @ee_hash = @hash.delete(:ee) || {} + + @hash[:tree] = normalize_tree(@hash[:tree]) + @ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {}) + end + # Returns a Hash of the YAML file, including EE specific data if EE is # used. def to_h - hash = parse_yaml - ee_hash = hash['ee'] - - if merge? && ee_hash - ee_hash.each do |key, value| - if key == 'project_tree' - merge_project_tree(value, hash[key]) - else - merge_attributes_list(value, hash[key]) - end - end + if merge_ee? + deep_merge(@hash, @ee_hash) + else + @hash end - - # We don't want to expose this section after this point, as it is no - # longer needed. - hash.delete('ee') - - hash end - # Merges a project relationships tree into the target tree. - # - # @param [Array<Hash|Symbol>] source_values - # @param [Array<Hash|Symbol>] target_values - def merge_project_tree(source_values, target_values) - source_values.each do |value| - if value.is_a?(Hash) - # Examples: - # - # { 'project_tree' => [{ 'labels' => [...] }] } - # { 'notes' => [:author, { 'events' => [:push_event_payload] }] } - value.each do |key, val| - target = target_values - .find { |h| h.is_a?(Hash) && h[key] } + private - if target - merge_project_tree(val, target[key]) - else - target_values << { key => val.dup } - end - end - else - # Example: :priorities, :author, etc - target_values << value - end + def deep_merge(hash_a, hash_b) + hash_a.deep_merge(hash_b) do |_, this_val, other_val| + this_val.to_a + other_val.to_a end end - # Merges a Hash containing a flat list of attributes, such as the entries - # in a `excluded_attributes` section. - # - # @param [Hash] source_values - # @param [Hash] target_values - def merge_attributes_list(source_values, target_values) - source_values.each do |key, values| - target_values[key] ||= [] - target_values[key].concat(values) + def normalize_tree(item) + case item + when Array + item.reduce({}) do |hash, subitem| + hash.merge!(normalize_tree(subitem)) + end + when Hash + item.transform_values(&method(:normalize_tree)) + when Symbol + { item => {} } + else + raise ArgumentError, "#{item} needs to be Array, Hash, Symbol or NilClass" end end - def merge? + def merge_ee? Gitlab.ee? end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 40acb24d191..06c94beead8 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -3,89 +3,92 @@ # This list _must_ only contain relationships that are available to both CE and # EE. EE specific relationships must be defined in the `ee` section further # down below. -project_tree: - - labels: - - :priorities - - milestones: - - events: - - :push_event_payload - - issues: - - events: - - :push_event_payload - - :timelogs - - notes: - - :author - - events: - - :push_event_payload - - label_links: - - label: - - :priorities - - milestone: - - events: - - :push_event_payload - - resource_label_events: - - label: - - :priorities - - :issue_assignees - - snippets: - - :award_emoji - - notes: - - :author - - releases: - - :links - - project_members: - - :user - - merge_requests: - - :metrics - - notes: - - :author +tree: + project: + - labels: + - :priorities + - milestones: - events: - :push_event_payload - - :suggestions - - merge_request_diff: - - :merge_request_diff_commits - - :merge_request_diff_files - - events: - - :push_event_payload - - :timelogs - - label_links: - - label: - - :priorities - - milestone: + - issues: - events: - :push_event_payload - - resource_label_events: - - label: - - :priorities - - ci_pipelines: - - notes: - - :author + - :timelogs + - notes: + - :author + - events: + - :push_event_payload + - label_links: + - label: + - :priorities + - milestone: + - events: + - :push_event_payload + - resource_label_events: + - label: + - :priorities + - :issue_assignees + - snippets: + - :award_emoji + - notes: + - :author + - releases: + - :links + - project_members: + - :user + - merge_requests: + - :metrics + - notes: + - :author + - events: + - :push_event_payload + - :suggestions + - merge_request_diff: + - :merge_request_diff_commits + - :merge_request_diff_files - events: - :push_event_payload - - stages: - - :statuses - - :external_pull_request - - :external_pull_requests - - :auto_devops - - :triggers - - :pipeline_schedules - - :services - - protected_branches: - - :merge_access_levels - - :push_access_levels - - protected_tags: - - :create_access_levels - - :project_feature - - :custom_attributes - - :prometheus_metrics - - :project_badges - - :ci_cd_settings - - :error_tracking_setting - - :metrics_setting - - boards: - - lists: - - label: - - :priorities + - :timelogs + - label_links: + - label: + - :priorities + - milestone: + - events: + - :push_event_payload + - resource_label_events: + - label: + - :priorities + - ci_pipelines: + - notes: + - :author + - events: + - :push_event_payload + - stages: + - :statuses + - :external_pull_request + - :external_pull_requests + - :auto_devops + - :triggers + - :pipeline_schedules + - :services + - protected_branches: + - :merge_access_levels + - :push_access_levels + - protected_tags: + - :create_access_levels + - :project_feature + - :custom_attributes + - :prometheus_metrics + - :project_badges + - :ci_cd_settings + - :error_tracking_setting + - :metrics_setting + - boards: + - lists: + - label: + - :priorities + group_members: + - :user # Only include the following attributes for the models specified. included_attributes: @@ -225,12 +228,15 @@ methods: - :type lists: - :list_type + ci_pipelines: + - :notes # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: - project_tree: - - protected_branches: - - :unprotect_access_levels - - protected_environments: - - :deploy_access_levels + tree: + project: + protected_branches: + - :unprotect_access_levels + protected_environments: + - :deploy_access_levels diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb deleted file mode 100644 index a92e3862361..00000000000 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json - # and its peculiar options. - class JsonHashBuilder - def self.build(model_objects, attributes_finder) - new(model_objects, attributes_finder).build - end - - def initialize(model_objects, attributes_finder) - @model_objects = model_objects - @attributes_finder = attributes_finder - end - - def build - process_model_objects(@model_objects) - end - - private - - # Called when the model is actually a hash containing other relations (more models) - # Returns the config in the right format for calling +to_json+ - # - # +model_object_hash+ - A model relationship such as: - # {:merge_requests=>[:merge_request_diff, :notes]} - def process_model_objects(model_object_hash) - json_config_hash = {} - current_key = model_object_hash.first.first - - model_object_hash.values.flatten.each do |model_object| - @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } - handle_model_object(current_key, model_object, json_config_hash) - end - - json_config_hash - end - - # Creates or adds to an existing hash an individual model or list - # - # +current_key+ main model that will be a key in the hash - # +model_object+ model or list of models to include in the hash - # +json_config_hash+ the original hash containing the root model - def handle_model_object(current_key, model_object, json_config_hash) - model_or_sub_model = model_object.is_a?(Hash) ? process_model_objects(model_object) : model_object - - if json_config_hash[current_key] - add_model_value(current_key, model_or_sub_model, json_config_hash) - else - create_model_value(current_key, model_or_sub_model, json_config_hash) - end - end - - # Constructs a new hash that will hold the configuration for that particular object - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - # +json_config_hash+ the original hash containing the root model - def create_model_value(current_key, value, json_config_hash) - json_config_hash[current_key] = parse_hash(value) || { include: value } - end - - # Calls attributes finder to parse the hash and add any attributes to it - # - # +value+ existing model to be included in the hash - # +parsed_hash+ the original hash - def parse_hash(value) - return if already_contains_methods?(value) - - @attributes_finder.parse(value) do |hash| - { include: hash_or_merge(value, hash) } - end - end - - def already_contains_methods?(value) - value.is_a?(Hash) && value.values.detect { |val| val[:methods]} - end - - # Adds new model configuration to an existing hash with key +current_key+ - # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ - # - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - # +json_config_hash+ the original hash containing the root model - def add_model_value(current_key, value, json_config_hash) - @attributes_finder.parse(value) do |hash| - value = { value => hash } unless value.is_a?(Hash) - end - - add_to_array(current_key, json_config_hash, value) - end - - # Adds new model configuration to an existing hash with key +current_key+ - # it creates a new array if it was previously a single value - # - # +current_key+ main model that will be a key in the hash - # +value+ existing model to be included in the hash - # +json_config_hash+ the original hash containing the root model - def add_to_array(current_key, json_config_hash, value) - old_values = json_config_hash[current_key][:include] - - json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten - end - - # Construct a new hash or merge with an existing one a model configuration - # This is to fulfil +to_json+ requirements. - # - # +hash+ hash containing configuration generated mainly from +@attributes_finder+ - # +value+ existing model to be included in the hash - def hash_or_merge(value, hash) - value.is_a?(Hash) ? value.merge(hash) : { value => hash } - end - end - end -end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 685cf53f27f..2dd18616cd6 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -58,11 +58,13 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - default_relation_list.each do |relation| - if relation.is_a?(Hash) - create_sub_relations(relation, @tree_hash) - elsif @tree_hash[relation.to_s].present? - save_relation_hash(@tree_hash[relation.to_s], relation) + project_relations_without_project_members.each do |relation_key, relation_definition| + relation_key_s = relation_key.to_s + + if relation_definition.present? + create_sub_relations(relation_key_s, relation_definition, @tree_hash) + elsif @tree_hash[relation_key_s].present? + save_relation_hash(relation_key_s, @tree_hash[relation_key_s]) end end @@ -71,7 +73,7 @@ module Gitlab @saved end - def save_relation_hash(relation_hash_batch, relation_key) + def save_relation_hash(relation_key, relation_hash_batch) relation_hash = create_relation(relation_key, relation_hash_batch) remove_group_models(relation_hash) if relation_hash.is_a?(Array) @@ -91,10 +93,13 @@ module Gitlab end end - def default_relation_list - reader.tree.reject do |model| - model.is_a?(Hash) && model[:project_members] - end + def project_relations_without_project_members + # We remove `project_members` as they are deserialized separately + project_relations.except(:project_members) + end + + def project_relations + reader.attributes_finder.find_relations_tree(:project) end def restore_project @@ -150,8 +155,7 @@ module Gitlab # issue, finds any subrelations such as notes, creates them and assign them back to the hash # # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation, tree_hash, save: true) - relation_key = relation.keys.first.to_s + def create_sub_relations(relation_key, relation_definition, tree_hash, save: true) return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten @@ -171,13 +175,13 @@ module Gitlab # But we can't have it in the upper level or GC won't get rid of the AR objects # after we save the batch. Project.transaction do - process_sub_relation(relation, relation_item) + process_sub_relation(relation_key, relation_definition, relation_item) # For every subrelation that hangs from Project, save the associated records altogether # This effectively batches all records per subrelation item, only keeping those in memory # We have to keep in mind that more batch granularity << Memory, but >> Slowness if save - save_relation_hash([relation_item], relation_key) + save_relation_hash(relation_key, [relation_item]) tree_hash[relation_key].delete(relation_item) end end @@ -186,37 +190,35 @@ module Gitlab tree_hash.delete(relation_key) if save end - def process_sub_relation(relation, relation_item) - relation.values.flatten.each do |sub_relation| + def process_sub_relation(relation_key, relation_definition, relation_item) + relation_definition.each do |sub_relation_key, sub_relation_definition| # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation == :author + next if sub_relation_key == :author - create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash) + sub_relation_key_s = sub_relation_key.to_s - relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) - relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? - end - end + # create dependent relations if present + if sub_relation_definition.present? + create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false) + end - def assign_relation_hash(relation_item, sub_relation) - if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] - sub_relation = sub_relation.keys.first - else - relation_hash = relation_item[sub_relation.to_s] + # transform relation hash to actual object + sub_relation_hash = relation_item[sub_relation_key_s] + if sub_relation_hash.present? + relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash) + end end - - [relation_hash, sub_relation] end - def create_relation(relation, relation_hash_list) + def create_relation(relation_key, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash, - members_mapper: members_mapper, - user: @user, - project: @restored_project, - excluded_keys: excluded_keys_for_relation(relation)) + Gitlab::ImportExport::RelationFactory.create( + relation_sym: relation_key.to_sym, + relation_hash: relation_hash, + members_mapper: members_mapper, + user: @user, + project: @restored_project, + excluded_keys: excluded_keys_for_relation(relation_key)) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2255635acdf..f1b3db6b208 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -18,7 +18,10 @@ module Gitlab def save mkdir_p(@shared.export_path) - File.write(full_path, project_json_tree) + project_tree = serialize_project_tree + fix_project_tree(project_tree) + File.write(full_path, project_tree.to_json) + true rescue => e @shared.error(e) @@ -27,27 +30,25 @@ module Gitlab private - def project_json_tree + def fix_project_tree(project_tree) if @params[:description].present? - project_json['description'] = @params[:description] + project_tree['description'] = @params[:description] end - project_json['project_members'] += group_members_json - - RelationRenameService.add_new_associations(project_json) + project_tree['project_members'] += group_members_array - project_json.to_json + RelationRenameService.add_new_associations(project_tree) end - def project_json - @project_json ||= @project.as_json(reader.project_tree) + def serialize_project_tree + @project.as_json(reader.project_tree) end def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end - def group_members_json + def group_members_array group_members.as_json(reader.group_members_tree).each do |group_member| group_member['source_type'] = 'Project' # Make group members project members of the future import end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 8bdf6ca491d..9e81c6a3d07 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -7,42 +7,22 @@ module Gitlab def initialize(shared:) @shared = shared - config_hash = ImportExport::Config.new.to_h.deep_symbolize_keys - @tree = config_hash[:project_tree] - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], - excluded_attributes: config_hash[:excluded_attributes], - methods: config_hash[:methods]) + + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new( + config: ImportExport::Config.new.to_h) end # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - attributes = @attributes_finder.find(:project) - project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} - - project_attributes.merge(include: build_hash(@tree)) + attributes_finder.find_root(:project) rescue => e @shared.error(e) false end def group_members_tree - @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user)) - end - - private - - # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html - # - # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file - def build_hash(model_list) - model_list.map do |model_objects| - if model_objects.is_a?(Hash) - Gitlab::ImportExport::JsonHashBuilder.build(model_objects, @attributes_finder) - else - @attributes_finder.find(model_objects) - end - end + attributes_finder.find_root(:group_members) end end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index a2296d265cd..93c0f3132d0 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -56,19 +56,23 @@ module Gitlab next if [namespace, key].any?(&:nil?) - options[namespace] ||= HashWithIndifferentAccess.new - - if option_multi_value?(namespace, key) - options[namespace][key] ||= HashWithIndifferentAccess.new(0) - options[namespace][key][value] += 1 - else - options[namespace][key] = value - end + store_option_info(options, namespace, key, value) end options end + def store_option_info(options, namespace, key, value) + options[namespace] ||= HashWithIndifferentAccess.new + + if option_multi_value?(namespace, key) + options[namespace][key] ||= HashWithIndifferentAccess.new(0) + options[namespace][key][value] += 1 + else + options[namespace][key] = value + end + end + def option_multi_value?(namespace, key) MULTI_VALUE_OPTIONS.any? { |arr| arr == [namespace, key] } end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index f2277dc3446..a77d70a5700 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -810,6 +810,9 @@ msgstr "" msgid "AdminArea|You’re about to stop all jobs.This will halt all current jobs that are running." msgstr "" +msgid "AdminDashboard|Error loading the statistics. Please try again" +msgstr "" + msgid "AdminProjects| You’re about to permanently delete the project %{projectName}, its repository, and all related resources including issues, merge requests, etc.. Once you confirm and press %{strong_start}Delete project%{strong_end}, it cannot be undone or recovered." msgstr "" @@ -837,6 +840,30 @@ msgstr "" msgid "AdminSettings|When creating a new environment variable it will be protected by default." msgstr "" +msgid "AdminStatistics|Active Users" +msgstr "" + +msgid "AdminStatistics|Forks" +msgstr "" + +msgid "AdminStatistics|Issues" +msgstr "" + +msgid "AdminStatistics|Merge Requests" +msgstr "" + +msgid "AdminStatistics|Milestones" +msgstr "" + +msgid "AdminStatistics|Notes" +msgstr "" + +msgid "AdminStatistics|SSH Keys" +msgstr "" + +msgid "AdminStatistics|Snippets" +msgstr "" + msgid "AdminUsers|2FA Disabled" msgstr "" @@ -10982,6 +11009,9 @@ msgstr "" msgid "State your message to activate" msgstr "" +msgid "Statistics" +msgstr "" + msgid "Status" msgstr "" diff --git a/qa/README.md b/qa/README.md index dede3cd2473..332e5c8170f 100644 --- a/qa/README.md +++ b/qa/README.md @@ -36,7 +36,7 @@ using `package-and-qa-manual` manual action, to test if everything works fine. You can use GitLab QA to exercise tests on any live instance! If you don't have an instance available you can follow the instructions below to use -the [GitLab Development Kit (GDK)][GDK]. +the [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit). This is the recommended option if you would like to contribute to the tests. Note: GitLab QA uses [Selenium WebDriver](https://www.seleniumhq.org/) via @@ -146,8 +146,6 @@ directory** (one level up from this directory): docker build -t gitlab/gitlab-ce-qa:nightly --file ./qa/Dockerfile ./ ``` -[GDK]: https://gitlab.com/gitlab-org/gitlab-development-kit/ - ### Quarantined tests Tests can be put in quarantine by assigning `:quarantine` metadata. This means diff --git a/spec/features/admin/dashboard_spec.rb b/spec/features/admin/dashboard_spec.rb index e204e0a515d..6cb345c5066 100644 --- a/spec/features/admin/dashboard_spec.rb +++ b/spec/features/admin/dashboard_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -describe 'admin visits dashboard' do +describe 'admin visits dashboard', :js do include ProjectForksHelper before do diff --git a/spec/fixtures/api/schemas/statistics.json b/spec/fixtures/api/schemas/statistics.json new file mode 100644 index 00000000000..ef2f39aad9d --- /dev/null +++ b/spec/fixtures/api/schemas/statistics.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "required" : [ + "forks", + "issues", + "merge_requests", + "notes", + "snippets", + "ssh_keys", + "milestones", + "users", + "projects", + "groups", + "active_users" + ], + "properties" : { + "forks": { "type": "string" }, + "issues'": { "type": "string" }, + "merge_requests'": { "type": "string" }, + "notes'": { "type": "string" }, + "snippets'": { "type": "string" }, + "ssh_keys'": { "type": "string" }, + "milestones'": { "type": "string" }, + "users'": { "type": "string" }, + "projects'": { "type": "string" }, + "groups'": { "type": "string" }, + "active_users'": { "type": "string" } + } +} diff --git a/spec/frontend/admin/statistics_panel/components/app_spec.js b/spec/frontend/admin/statistics_panel/components/app_spec.js new file mode 100644 index 00000000000..25b1d432e2d --- /dev/null +++ b/spec/frontend/admin/statistics_panel/components/app_spec.js @@ -0,0 +1,73 @@ +import Vuex from 'vuex'; +import { createLocalVue, shallowMount } from '@vue/test-utils'; +import AxiosMockAdapter from 'axios-mock-adapter'; +import axios from '~/lib/utils/axios_utils'; +import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; +import StatisticsPanelApp from '~/admin/statistics_panel/components/app.vue'; +import statisticsLabels from '~/admin/statistics_panel/constants'; +import createStore from '~/admin/statistics_panel/store'; +import { GlLoadingIcon } from '@gitlab/ui'; +import mockStatistics from '../mock_data'; + +const localVue = createLocalVue(); +localVue.use(Vuex); + +describe('Admin statistics app', () => { + let wrapper; + let store; + let axiosMock; + + const createComponent = () => { + wrapper = shallowMount(StatisticsPanelApp, { + localVue, + store, + sync: false, + }); + }; + + beforeEach(() => { + axiosMock = new AxiosMockAdapter(axios); + axiosMock.onGet(/api\/(.*)\/application\/statistics/).reply(200); + store = createStore(); + }); + + afterEach(() => { + wrapper.destroy(); + }); + + const findStats = idx => wrapper.findAll('.js-stats').at(idx); + + describe('template', () => { + describe('when app is loading', () => { + it('renders a loading indicator', () => { + store.dispatch('requestStatistics'); + createComponent(); + + expect(wrapper.find(GlLoadingIcon).exists()).toBe(true); + }); + }); + + describe('when app has finished loading', () => { + const statistics = convertObjectPropsToCamelCase(mockStatistics, { deep: true }); + + it.each` + statistic | count | index + ${'forks'} | ${12} | ${0} + ${'issues'} | ${180} | ${1} + ${'mergeRequests'} | ${31} | ${2} + ${'notes'} | ${986} | ${3} + ${'snippets'} | ${50} | ${4} + ${'sshKeys'} | ${10} | ${5} + ${'milestones'} | ${40} | ${6} + ${'activeUsers'} | ${50} | ${7} + `('renders the count for the $statistic statistic', ({ statistic, count, index }) => { + const label = statisticsLabels[statistic]; + store.dispatch('receiveStatisticsSuccess', statistics); + createComponent(); + + expect(findStats(index).text()).toContain(label); + expect(findStats(index).text()).toContain(count); + }); + }); + }); +}); diff --git a/spec/frontend/admin/statistics_panel/mock_data.js b/spec/frontend/admin/statistics_panel/mock_data.js new file mode 100644 index 00000000000..6d861059dfd --- /dev/null +++ b/spec/frontend/admin/statistics_panel/mock_data.js @@ -0,0 +1,15 @@ +const mockStatistics = { + forks: 12, + issues: 180, + merge_requests: 31, + notes: 986, + snippets: 50, + ssh_keys: 10, + milestones: 40, + users: 50, + projects: 29, + groups: 9, + active_users: 50, +}; + +export default mockStatistics; diff --git a/spec/frontend/admin/statistics_panel/store/actions_spec.js b/spec/frontend/admin/statistics_panel/store/actions_spec.js new file mode 100644 index 00000000000..9b18b1aebda --- /dev/null +++ b/spec/frontend/admin/statistics_panel/store/actions_spec.js @@ -0,0 +1,115 @@ +import axios from 'axios'; +import MockAdapter from 'axios-mock-adapter'; +import testAction from 'helpers/vuex_action_helper'; +import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; +import * as actions from '~/admin/statistics_panel/store/actions'; +import * as types from '~/admin/statistics_panel/store/mutation_types'; +import getInitialState from '~/admin/statistics_panel/store/state'; +import mockStatistics from '../mock_data'; + +describe('Admin statistics panel actions', () => { + let mock; + let state; + + beforeEach(() => { + state = getInitialState(); + mock = new MockAdapter(axios); + }); + + describe('fetchStatistics', () => { + describe('success', () => { + beforeEach(() => { + mock.onGet(/api\/(.*)\/application\/statistics/).replyOnce(200, mockStatistics); + }); + + it('dispatches success with received data', done => + testAction( + actions.fetchStatistics, + null, + state, + [], + [ + { type: 'requestStatistics' }, + { + type: 'receiveStatisticsSuccess', + payload: expect.objectContaining( + convertObjectPropsToCamelCase(mockStatistics, { deep: true }), + ), + }, + ], + done, + )); + }); + + describe('error', () => { + beforeEach(() => { + mock.onGet(/api\/(.*)\/application\/statistics/).replyOnce(500); + }); + + it('dispatches error', done => + testAction( + actions.fetchStatistics, + null, + state, + [], + [ + { + type: 'requestStatistics', + }, + { + type: 'receiveStatisticsError', + payload: new Error('Request failed with status code 500'), + }, + ], + done, + )); + }); + }); + + describe('requestStatistic', () => { + it('should commit the request mutation', done => + testAction( + actions.requestStatistics, + null, + state, + [{ type: types.REQUEST_STATISTICS }], + [], + done, + )); + }); + + describe('receiveStatisticsSuccess', () => { + it('should commit received data', done => + testAction( + actions.receiveStatisticsSuccess, + mockStatistics, + state, + [ + { + type: types.RECEIVE_STATISTICS_SUCCESS, + payload: mockStatistics, + }, + ], + [], + done, + )); + }); + + describe('receiveStatisticsError', () => { + it('should commit error', done => { + testAction( + actions.receiveStatisticsError, + 500, + state, + [ + { + type: types.RECEIVE_STATISTICS_ERROR, + payload: 500, + }, + ], + [], + done, + ); + }); + }); +}); diff --git a/spec/frontend/admin/statistics_panel/store/getters_spec.js b/spec/frontend/admin/statistics_panel/store/getters_spec.js new file mode 100644 index 00000000000..152d82531ed --- /dev/null +++ b/spec/frontend/admin/statistics_panel/store/getters_spec.js @@ -0,0 +1,48 @@ +import createState from '~/admin/statistics_panel/store/state'; +import * as getters from '~/admin/statistics_panel/store/getters'; + +describe('Admin statistics panel getters', () => { + let state; + + beforeEach(() => { + state = createState(); + }); + + describe('getStatistics', () => { + describe('when statistics data exists', () => { + it('returns an array of statistics objects with key, label and value', () => { + state.statistics = { forks: 10, issues: 20 }; + + const statisticsLabels = { + forks: 'Forks', + issues: 'Issues', + }; + + const statisticsData = [ + { key: 'forks', label: 'Forks', value: 10 }, + { key: 'issues', label: 'Issues', value: 20 }, + ]; + + expect(getters.getStatistics(state)(statisticsLabels)).toEqual(statisticsData); + }); + }); + + describe('when no statistics data exists', () => { + it('returns an array of statistics objects with key, label and sets value to null', () => { + state.statistics = null; + + const statisticsLabels = { + forks: 'Forks', + issues: 'Issues', + }; + + const statisticsData = [ + { key: 'forks', label: 'Forks', value: null }, + { key: 'issues', label: 'Issues', value: null }, + ]; + + expect(getters.getStatistics(state)(statisticsLabels)).toEqual(statisticsData); + }); + }); + }); +}); diff --git a/spec/frontend/admin/statistics_panel/store/mutations_spec.js b/spec/frontend/admin/statistics_panel/store/mutations_spec.js new file mode 100644 index 00000000000..179f38d2bc5 --- /dev/null +++ b/spec/frontend/admin/statistics_panel/store/mutations_spec.js @@ -0,0 +1,41 @@ +import mutations from '~/admin/statistics_panel/store/mutations'; +import * as types from '~/admin/statistics_panel/store/mutation_types'; +import getInitialState from '~/admin/statistics_panel/store/state'; +import mockStatistics from '../mock_data'; + +describe('Admin statistics panel mutations', () => { + let state; + + beforeEach(() => { + state = getInitialState(); + }); + + describe(`${types.REQUEST_STATISTICS}`, () => { + it('sets isLoading to true', () => { + mutations[types.REQUEST_STATISTICS](state); + + expect(state.isLoading).toBe(true); + }); + }); + + describe(`${types.RECEIVE_STATISTICS_SUCCESS}`, () => { + it('updates the store with the with statistics', () => { + mutations[types.RECEIVE_STATISTICS_SUCCESS](state, mockStatistics); + + expect(state.isLoading).toBe(false); + expect(state.error).toBe(null); + expect(state.statistics).toEqual(mockStatistics); + }); + }); + + describe(`${types.RECEIVE_STATISTICS_ERROR}`, () => { + it('sets error and clears data', () => { + const error = 500; + mutations[types.RECEIVE_STATISTICS_ERROR](state, error); + + expect(state.isLoading).toBe(false); + expect(state.error).toBe(error); + expect(state.statistics).toEqual(null); + }); + }); +}); diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 89431b80be3..023d7530b4b 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -46,7 +46,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do context 'is matched' do let(:attributes) { { name: 'rspec', ref: 'master', rules: [{ if: '$VAR == null', when: 'delayed', start_in: '3 hours' }] } } - it { is_expected.to include(when: 'delayed', start_in: '3 hours') } + it { is_expected.to include(when: 'delayed', options: { start_in: '3 hours' }) } end context 'is not matched' do @@ -541,7 +541,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do it { is_expected.to be_included } it 'correctly populates when:' do - expect(seed_build.attributes).to include(when: 'delayed', start_in: '1 day') + expect(seed_build.attributes).to include(when: 'delayed', options: { start_in: '1 day' }) end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index cf496b79a62..9d9a9ecda33 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -16,7 +16,10 @@ module Gitlab let(:config) do YAML.dump( before_script: ['pwd'], - rspec: { script: 'rspec' } + rspec: { + script: 'rspec', + interruptible: true + } ) end @@ -29,6 +32,7 @@ module Gitlab before_script: ["pwd"], script: ["rspec"] }, + interruptible: true, allow_failure: false, when: "on_success", yaml_variables: [] @@ -36,6 +40,36 @@ module Gitlab end end + context 'with job rules' do + let(:config) do + YAML.dump( + rspec: { + script: 'rspec', + rules: [ + { if: '$CI_COMMIT_REF_NAME == "master"' }, + { changes: %w[README.md] } + ] + } + ) + end + + it 'returns valid build attributes' do + expect(subject).to eq({ + stage: 'test', + stage_idx: 1, + name: 'rspec', + options: { script: ['rspec'] }, + rules: [ + { if: '$CI_COMMIT_REF_NAME == "master"' }, + { changes: %w[README.md] } + ], + allow_failure: false, + when: 'on_success', + yaml_variables: [] + }) + end + end + describe 'coverage entry' do describe 'code coverage regexp' do let(:config) do @@ -1252,7 +1286,7 @@ module Gitlab end end - describe 'rules' do + context 'with when/rules conflict' do subject { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } let(:config) do diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb index fef84c87509..cc8ca1d87e3 100644 --- a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -12,7 +12,7 @@ describe 'Import/Export attribute configuration' do let(:config_hash) { Gitlab::ImportExport::Config.new.to_h.deep_stringify_keys } let(:relation_names) do - names = names_from_tree(config_hash['project_tree']) + names = names_from_tree(config_hash.dig('tree', 'project')) # Remove duplicated or add missing models # - project is not part of the tree, so it has to be added manually. diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb new file mode 100644 index 00000000000..208b60844e3 --- /dev/null +++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +describe Gitlab::ImportExport::AttributesFinder do + describe '#find_root' do + subject { described_class.new(config: config).find_root(model_key) } + + let(:test_config) { 'spec/support/import_export/import_export.yml' } + let(:config) { Gitlab::ImportExport::Config.new.to_h } + let(:model_key) { :project } + + let(:project_tree_hash) do + { + except: [:id, :created_at], + include: [ + { issues: { include: [] } }, + { labels: { include: [] } }, + { merge_requests: { + except: [:iid], + include: [ + { merge_request_diff: { + include: [] + } }, + { merge_request_test: { include: [] } } + ], + only: [:id] + } }, + { commit_statuses: { + include: [{ commit: { include: [] } }] + } }, + { project_members: { + include: [{ user: { include: [], + only: [:email] } }] + } } + ] + } + end + + before do + allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config) + end + + it 'generates hash from project tree config' do + is_expected.to match(project_tree_hash) + end + + context 'individual scenarios' do + it 'generates the correct hash for a single project relation' do + setup_yaml(tree: { project: [:issues] }) + + is_expected.to match( + include: [{ issues: { include: [] } }] + ) + end + + it 'generates the correct hash for a single project feature relation' do + setup_yaml(tree: { project: [:project_feature] }) + + is_expected.to match( + include: [{ project_feature: { include: [] } }] + ) + end + + it 'generates the correct hash for a multiple project relation' do + setup_yaml(tree: { project: [:issues, :snippets] }) + + is_expected.to match( + include: [{ issues: { include: [] } }, + { snippets: { include: [] } }] + ) + end + + it 'generates the correct hash for a single sub-relation' do + setup_yaml(tree: { project: [issues: [:notes]] }) + + is_expected.to match( + include: [{ issues: { include: [{ notes: { include: [] } }] } }] + ) + end + + it 'generates the correct hash for a multiple sub-relation' do + setup_yaml(tree: { project: [merge_requests: [:notes, :merge_request_diff]] }) + + is_expected.to match( + include: [{ merge_requests: + { include: [{ notes: { include: [] } }, + { merge_request_diff: { include: [] } }] } }] + ) + end + + it 'generates the correct hash for a sub-relation with another sub-relation' do + setup_yaml(tree: { project: [merge_requests: [notes: [:author]]] }) + + is_expected.to match( + include: [{ merge_requests: { + include: [{ notes: { include: [{ author: { include: [] } }] } }] + } }] + ) + end + + it 'generates the correct hash for a relation with included attributes' do + setup_yaml(tree: { project: [:issues] }, + included_attributes: { issues: [:name, :description] }) + + is_expected.to match( + include: [{ issues: { include: [], + only: [:name, :description] } }] + ) + end + + it 'generates the correct hash for a relation with excluded attributes' do + setup_yaml(tree: { project: [:issues] }, + excluded_attributes: { issues: [:name] }) + + is_expected.to match( + include: [{ issues: { except: [:name], + include: [] } }] + ) + end + + it 'generates the correct hash for a relation with both excluded and included attributes' do + setup_yaml(tree: { project: [:issues] }, + excluded_attributes: { issues: [:name] }, + included_attributes: { issues: [:description] }) + + is_expected.to match( + include: [{ issues: { except: [:name], + include: [], + only: [:description] } }] + ) + end + + it 'generates the correct hash for a relation with custom methods' do + setup_yaml(tree: { project: [:issues] }, + methods: { issues: [:name] }) + + is_expected.to match( + include: [{ issues: { include: [], + methods: [:name] } }] + ) + end + + def setup_yaml(hash) + allow(YAML).to receive(:load_file).with(test_config).and_return(hash) + end + end + end + + describe '#find_relations_tree' do + subject { described_class.new(config: config).find_relations_tree(model_key) } + + let(:tree) { { project: { issues: {} } } } + let(:model_key) { :project } + + context 'when initialized with config including tree' do + let(:config) { { tree: tree } } + + context 'when relation is in top-level keys of the tree' do + it { is_expected.to eq({ issues: {} }) } + end + + context 'when the relation is not in top-level keys' do + let(:model_key) { :issues } + + it { is_expected.to be_nil } + end + end + + context 'when tree is not present in config' do + let(:config) { {} } + + it { is_expected.to be_nil } + end + end + + describe '#find_excluded_keys' do + subject { described_class.new(config: config).find_excluded_keys(klass_name) } + + let(:klass_name) { 'project' } + + context 'when initialized with excluded_attributes' do + let(:config) { { excluded_attributes: excluded_attributes } } + let(:excluded_attributes) { { project: [:name, :path], issues: [:milestone_id] } } + + it { is_expected.to eq(%w[name path]) } + end + + context 'when excluded_attributes are not present in config' do + let(:config) { {} } + + it { is_expected.to eq([]) } + end + end +end diff --git a/spec/lib/gitlab/import_export/config_spec.rb b/spec/lib/gitlab/import_export/config_spec.rb index cf396dba382..e53db37def4 100644 --- a/spec/lib/gitlab/import_export/config_spec.rb +++ b/spec/lib/gitlab/import_export/config_spec.rb @@ -1,163 +1,159 @@ # frozen_string_literal: true -require 'spec_helper' +require 'fast_spec_helper' +require 'rspec-parameterized' describe Gitlab::ImportExport::Config do let(:yaml_file) { described_class.new } describe '#to_h' do - context 'when using CE' do - before do - allow(yaml_file) - .to receive(:merge?) - .and_return(false) + subject { yaml_file.to_h } + + context 'when using default config' do + using RSpec::Parameterized::TableSyntax + + where(:ee) do + [true, false] end - it 'just returns the parsed Hash without the EE section' do - expected = YAML.load_file(Gitlab::ImportExport.config_file) - expected.delete('ee') + with_them do + before do + allow(Gitlab).to receive(:ee?) { ee } + end - expect(yaml_file.to_h).to eq(expected) + it 'parses default config' do + expect { subject }.not_to raise_error + expect(subject).to be_a(Hash) + expect(subject.keys).to contain_exactly( + :tree, :excluded_attributes, :included_attributes, :methods) + end end end - context 'when using EE' do - before do - allow(yaml_file) - .to receive(:merge?) - .and_return(true) - end + context 'when using custom config' do + let(:config) do + <<-EOF.strip_heredoc + tree: + project: + - labels: + - :priorities + - milestones: + - events: + - :push_event_payload - it 'merges the EE project tree into the CE project tree' do - allow(yaml_file) - .to receive(:parse_yaml) - .and_return({ - 'project_tree' => [ - { - 'issues' => [ - :id, - :title, - { 'notes' => [:id, :note, { 'author' => [:name] }] } - ] - } - ], - 'ee' => { - 'project_tree' => [ - { - 'issues' => [ - :description, - { 'notes' => [:date, { 'author' => [:email] }] } - ] - }, - { 'foo' => [{ 'bar' => %i[baz] }] } - ] - } - }) + included_attributes: + user: + - :id - expect(yaml_file.to_h).to eq({ - 'project_tree' => [ - { - 'issues' => [ - :id, - :title, - { - 'notes' => [ - :id, - :note, - { 'author' => [:name, :email] }, - :date - ] - }, - :description - ] - }, - { 'foo' => [{ 'bar' => %i[baz] }] } - ] - }) + excluded_attributes: + project: + - :name + + methods: + labels: + - :type + events: + - :action + + ee: + tree: + project: + protected_branches: + - :unprotect_access_levels + included_attributes: + user: + - :name_ee + excluded_attributes: + project: + - :name_without_ee + methods: + labels: + - :type_ee + events_ee: + - :action_ee + EOF end - it 'merges the excluded attributes list' do - allow(yaml_file) - .to receive(:parse_yaml) - .and_return({ - 'project_tree' => [], - 'excluded_attributes' => { - 'project' => %i[id title], - 'notes' => %i[id] - }, - 'ee' => { - 'project_tree' => [], - 'excluded_attributes' => { - 'project' => %i[date], - 'foo' => %i[bar baz] - } - } - }) - - expect(yaml_file.to_h).to eq({ - 'project_tree' => [], - 'excluded_attributes' => { - 'project' => %i[id title date], - 'notes' => %i[id], - 'foo' => %i[bar baz] - } - }) + let(:config_hash) { YAML.safe_load(config, [Symbol]) } + + before do + allow_any_instance_of(described_class).to receive(:parse_yaml) do + config_hash.deep_dup + end end - it 'merges the included attributes list' do - allow(yaml_file) - .to receive(:parse_yaml) - .and_return({ - 'project_tree' => [], - 'included_attributes' => { - 'project' => %i[id title], - 'notes' => %i[id] - }, - 'ee' => { - 'project_tree' => [], - 'included_attributes' => { - 'project' => %i[date], - 'foo' => %i[bar baz] + context 'when using CE' do + before do + allow(Gitlab).to receive(:ee?) { false } + end + + it 'just returns the normalized Hash' do + is_expected.to eq( + { + tree: { + project: { + labels: { + priorities: {} + }, + milestones: { + events: { + push_event_payload: {} + } + } + } + }, + included_attributes: { + user: [:id] + }, + excluded_attributes: { + project: [:name] + }, + methods: { + labels: [:type], + events: [:action] } } - }) - - expect(yaml_file.to_h).to eq({ - 'project_tree' => [], - 'included_attributes' => { - 'project' => %i[id title date], - 'notes' => %i[id], - 'foo' => %i[bar baz] - } - }) + ) + end end - it 'merges the methods list' do - allow(yaml_file) - .to receive(:parse_yaml) - .and_return({ - 'project_tree' => [], - 'methods' => { - 'project' => %i[id title], - 'notes' => %i[id] - }, - 'ee' => { - 'project_tree' => [], - 'methods' => { - 'project' => %i[date], - 'foo' => %i[bar baz] + context 'when using EE' do + before do + allow(Gitlab).to receive(:ee?) { true } + end + + it 'just returns the normalized Hash' do + is_expected.to eq( + { + tree: { + project: { + labels: { + priorities: {} + }, + milestones: { + events: { + push_event_payload: {} + } + }, + protected_branches: { + unprotect_access_levels: {} + } + } + }, + included_attributes: { + user: [:id, :name_ee] + }, + excluded_attributes: { + project: [:name, :name_without_ee] + }, + methods: { + labels: [:type, :type_ee], + events: [:action], + events_ee: [:action_ee] } } - }) - - expect(yaml_file.to_h).to eq({ - 'project_tree' => [], - 'methods' => { - 'project' => %i[id title date], - 'notes' => %i[id], - 'foo' => %i[bar baz] - } - }) + ) + end end end end diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index 5ed9fef1597..3442e22c11f 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -8,7 +8,7 @@ describe 'Import/Export model configuration' do let(:config_hash) { Gitlab::ImportExport::Config.new.to_h.deep_stringify_keys } let(:model_names) do - names = names_from_tree(config_hash['project_tree']) + names = names_from_tree(config_hash.dig('tree', 'project')) # Remove duplicated or add missing models # - project is not part of the tree, so it has to be added manually. diff --git a/spec/lib/gitlab/import_export/reader_spec.rb b/spec/lib/gitlab/import_export/reader_spec.rb index f93ff074770..87f665bd995 100644 --- a/spec/lib/gitlab/import_export/reader_spec.rb +++ b/spec/lib/gitlab/import_export/reader_spec.rb @@ -2,96 +2,45 @@ require 'spec_helper' describe Gitlab::ImportExport::Reader do let(:shared) { Gitlab::ImportExport::Shared.new(nil) } - let(:test_config) { 'spec/support/import_export/import_export.yml' } - let(:project_tree_hash) do - { - except: [:id, :created_at], - include: [:issues, :labels, - { merge_requests: { - only: [:id], - except: [:iid], - include: [:merge_request_diff, :merge_request_test] - } }, - { commit_statuses: { include: :commit } }, - { project_members: { include: { user: { only: [:email] } } } }] - } - end - - before do - allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config) - end - - it 'generates hash from project tree config' do - expect(described_class.new(shared: shared).project_tree).to match(project_tree_hash) - end - - context 'individual scenarios' do - it 'generates the correct hash for a single project relation' do - setup_yaml(project_tree: [:issues]) - - expect(described_class.new(shared: shared).project_tree).to match(include: [:issues]) - end - - it 'generates the correct hash for a single project feature relation' do - setup_yaml(project_tree: [:project_feature]) - expect(described_class.new(shared: shared).project_tree).to match(include: [:project_feature]) - end + describe '#project_tree' do + subject { described_class.new(shared: shared).project_tree } - it 'generates the correct hash for a multiple project relation' do - setup_yaml(project_tree: [:issues, :snippets]) + it 'delegates to AttributesFinder#find_root' do + expect_any_instance_of(Gitlab::ImportExport::AttributesFinder) + .to receive(:find_root) + .with(:project) - expect(described_class.new(shared: shared).project_tree).to match(include: [:issues, :snippets]) + subject end - it 'generates the correct hash for a single sub-relation' do - setup_yaml(project_tree: [issues: [:notes]]) + context 'when exception raised' do + before do + expect_any_instance_of(Gitlab::ImportExport::AttributesFinder) + .to receive(:find_root) + .with(:project) + .and_raise(StandardError) + end - expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { include: :notes } }]) - end - - it 'generates the correct hash for a multiple sub-relation' do - setup_yaml(project_tree: [merge_requests: [:notes, :merge_request_diff]]) - - expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: [:notes, :merge_request_diff] } }]) - end + it { is_expected.to be false } - it 'generates the correct hash for a sub-relation with another sub-relation' do - setup_yaml(project_tree: [merge_requests: [notes: :author]]) + it 'logs the error' do + expect(shared).to receive(:error).with(instance_of(StandardError)) - expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: { notes: { include: :author } } } }]) + subject + end end + end - it 'generates the correct hash for a relation with included attributes' do - setup_yaml(project_tree: [:issues], included_attributes: { issues: [:name, :description] }) - - expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { only: [:name, :description] } }]) - end - - it 'generates the correct hash for a relation with excluded attributes' do - setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] }) - - expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name] } }]) - end - - it 'generates the correct hash for a relation with both excluded and included attributes' do - setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] }, included_attributes: { issues: [:description] }) - - expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name], only: [:description] } }]) - end - - it 'generates the correct hash for a relation with custom methods' do - setup_yaml(project_tree: [:issues], methods: { issues: [:name] }) - - expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { methods: [:name] } }]) - end + describe '#group_members_tree' do + subject { described_class.new(shared: shared).group_members_tree } - it 'generates the correct hash for group members' do - expect(described_class.new(shared: shared).group_members_tree).to match({ include: { user: { only: [:email] } } }) - end + it 'delegates to AttributesFinder#find_root' do + expect_any_instance_of(Gitlab::ImportExport::AttributesFinder) + .to receive(:find_root) + .with(:group_members) - def setup_yaml(hash) - allow(YAML).to receive(:load_file).with(test_config).and_return(hash) + subject end end end diff --git a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb index 15748407f0c..17bb5bcc155 100644 --- a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb +++ b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb @@ -12,7 +12,7 @@ describe Gitlab::ImportExport::RelationRenameService do let(:user) { create(:admin) } let(:group) { create(:group, :nested) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } + let!(:project) { create(:project, :builds_disabled, :issues_disabled, group: group, name: 'project', path: 'project') } let(:shared) { project.import_export_shared } before do @@ -24,7 +24,6 @@ describe Gitlab::ImportExport::RelationRenameService do let(:import_path) { 'spec/lib/gitlab/import_export' } let(:file_content) { IO.read("#{import_path}/project.json") } let!(:json_file) { ActiveSupport::JSON.decode(file_content) } - let(:tree_hash) { project_tree_restorer.instance_variable_get(:@tree_hash) } before do allow(shared).to receive(:export_path).and_return(import_path) @@ -92,21 +91,25 @@ describe Gitlab::ImportExport::RelationRenameService do end context 'when exporting' do - let(:project_tree_saver) { Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: user, shared: shared) } - let(:project_tree) { project_tree_saver.send(:project_json) } + let(:export_content_path) { project_tree_saver.full_path } + let(:export_content_hash) { ActiveSupport::JSON.decode(File.read(export_content_path)) } + let(:injected_hash) { renames.values.product([{}]).to_h } - it 'adds old relationships to the exported file' do - project_tree.merge!(renames.values.map { |new_name| [new_name, []] }.to_h) + let(:project_tree_saver) do + Gitlab::ImportExport::ProjectTreeSaver.new( + project: project, current_user: user, shared: shared) + end - allow(project_tree_saver).to receive(:save) do |arg| - project_tree_saver.send(:project_json_tree) + it 'adds old relationships to the exported file' do + # we inject relations with new names that should be rewritten + expect(project_tree_saver).to receive(:serialize_project_tree).and_wrap_original do |method, *args| + method.call(*args).merge(injected_hash) end - result = project_tree_saver.save - - saved_data = ActiveSupport::JSON.decode(result) + expect(project_tree_saver.save).to eq(true) - expect(saved_data.keys).to include(*(renames.keys + renames.values)) + expect(export_content_hash.keys).to include(*renames.keys) + expect(export_content_hash.keys).to include(*renames.values) end end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b8c323904b8..6722a3c627d 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -32,7 +32,7 @@ describe User do it { is_expected.to have_many(:groups) } it { is_expected.to have_many(:keys).dependent(:destroy) } it { is_expected.to have_many(:deploy_keys).dependent(:nullify) } - it { is_expected.to have_many(:events).dependent(:destroy) } + it { is_expected.to have_many(:events).dependent(:delete_all) } it { is_expected.to have_many(:issues).dependent(:destroy) } it { is_expected.to have_many(:notes).dependent(:destroy) } it { is_expected.to have_many(:merge_requests).dependent(:destroy) } diff --git a/spec/requests/api/statistics_spec.rb b/spec/requests/api/statistics_spec.rb new file mode 100644 index 00000000000..91fc4d4c123 --- /dev/null +++ b/spec/requests/api/statistics_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::Statistics, 'Statistics' do + include ProjectForksHelper + TABLES_TO_ANALYZE = %w[ + projects + users + namespaces + issues + merge_requests + notes + snippets + fork_networks + fork_network_members + keys + milestones + ].freeze + + let(:path) { "/application/statistics" } + + describe "GET /application/statistics" do + context 'when no user' do + it "returns authentication error" do + get api(path, nil) + + expect(response).to have_gitlab_http_status(401) + end + end + + context "when not an admin" do + let(:user) { create(:user) } + + it "returns forbidden error" do + get api(path, user) + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'when authenticated as admin' do + let(:admin) { create(:admin) } + + it 'matches the response schema' do + get api(path, admin) + + expect(response).to have_gitlab_http_status(200) + expect(response).to match_response_schema('statistics') + end + + it 'gives the right statistics' do + projects = create_list(:project, 4, namespace: create(:namespace, owner: admin)) + issues = create_list(:issue, 2, project: projects.first, updated_by: admin) + + create_list(:snippet, 2, :public, author: admin) + create_list(:note, 2, author: admin, project: projects.first, noteable: issues.first) + create_list(:milestone, 3, project: projects.first) + create(:key, user: admin) + create(:merge_request, source_project: projects.first) + fork_project(projects.first, admin) + + # Make sure the reltuples have been updated + # to get a correct count on postgresql + TABLES_TO_ANALYZE.each do |table| + ActiveRecord::Base.connection.execute("ANALYZE #{table}") + end + + get api(path, admin) + + expected_statistics = { + issues: 2, + merge_requests: 1, + notes: 2, + snippets: 2, + forks: 1, + ssh_keys: 1, + milestones: 3, + users: 1, + projects: 5, + groups: 1, + active_users: 1 + } + + expected_statistics.each do |entity, count| + expect(json_response[entity.to_s]).to eq(count.to_s) + end + end + end + end +end diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index d8880819d9f..fe86982af91 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -760,33 +760,32 @@ describe Ci::CreatePipelineService do end context 'when builds with auto-retries are configured' do + let(:pipeline) { execute_service } + let(:rspec_job) { pipeline.builds.find_by(name: 'rspec') } + + before do + stub_ci_pipeline_yaml_file(YAML.dump({ + rspec: { script: 'rspec', retry: retry_value } + })) + end + context 'as an integer' do - before do - config = YAML.dump(rspec: { script: 'rspec', retry: 2 }) - stub_ci_pipeline_yaml_file(config) - end + let(:retry_value) { 2 } it 'correctly creates builds with auto-retry value configured' do - pipeline = execute_service - expect(pipeline).to be_persisted - expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2 - expect(pipeline.builds.find_by(name: 'rspec').retry_when).to eq ['always'] + expect(rspec_job.retries_max).to eq 2 + expect(rspec_job.retry_when).to eq ['always'] end end context 'as hash' do - before do - config = YAML.dump(rspec: { script: 'rspec', retry: { max: 2, when: 'runner_system_failure' } }) - stub_ci_pipeline_yaml_file(config) - end + let(:retry_value) { { max: 2, when: 'runner_system_failure' } } it 'correctly creates builds with auto-retry value configured' do - pipeline = execute_service - expect(pipeline).to be_persisted - expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2 - expect(pipeline.builds.find_by(name: 'rspec').retry_when).to eq ['runner_system_failure'] + expect(rspec_job.retries_max).to eq 2 + expect(rspec_job.retry_when).to eq ['runner_system_failure'] end end end @@ -1174,7 +1173,7 @@ describe Ci::CreatePipelineService do expect(pipeline).to be_persisted expect(pipeline).to be_merge_request_event expect(pipeline.merge_request).to eq(merge_request) - expect(pipeline.builds.order(:stage_id).map(&:name)).to eq(%w[test]) + expect(pipeline.builds.order(:stage_id).pluck(:name)).to eq(%w[test]) end it 'persists the specified source sha' do @@ -1439,7 +1438,7 @@ describe Ci::CreatePipelineService do expect(pipeline).to be_persisted expect(pipeline).to be_web expect(pipeline.merge_request).to be_nil - expect(pipeline.builds.order(:stage_id).map(&:name)).to eq(%w[build pages]) + expect(pipeline.builds.order(:stage_id).pluck(:name)).to eq(%w[build pages]) end end end @@ -1479,7 +1478,7 @@ describe Ci::CreatePipelineService do it 'creates a pipeline with build_a and test_a' do expect(pipeline).to be_persisted - expect(pipeline.builds.map(&:name)).to contain_exactly("build_a", "test_a") + expect(pipeline.builds.pluck(:name)).to contain_exactly("build_a", "test_a") end end @@ -1514,7 +1513,303 @@ describe Ci::CreatePipelineService do it 'does create a pipeline only with deploy' do expect(pipeline).to be_persisted - expect(pipeline.builds.map(&:name)).to contain_exactly("deploy") + expect(pipeline.builds.pluck(:name)).to contain_exactly("deploy") + end + end + end + + context 'when rules are used' do + let(:ref_name) { 'refs/heads/master' } + let(:pipeline) { execute_service } + let(:build_names) { pipeline.builds.pluck(:name) } + let(:regular_job) { pipeline.builds.find_by(name: 'regular-job') } + let(:rules_job) { pipeline.builds.find_by(name: 'rules-job') } + let(:delayed_job) { pipeline.builds.find_by(name: 'delayed-job') } + + shared_examples 'rules jobs are excluded' do + it 'only persists the job without rules' do + expect(pipeline).to be_persisted + expect(regular_job).to be_persisted + expect(rules_job).to be_nil + expect(delayed_job).to be_nil + end + end + + before do + stub_ci_pipeline_yaml_file(config) + allow_any_instance_of(Ci::BuildScheduleWorker).to receive(:perform).and_return(true) + end + + context 'with simple if: clauses' do + let(:config) do + <<-EOY + regular-job: + script: 'echo Hello, World!' + + master-job: + script: "echo hello world, $CI_COMMIT_REF_NAME" + rules: + - if: $CI_COMMIT_REF_NAME == "nonexistant-branch" + when: never + - if: $CI_COMMIT_REF_NAME =~ /master/ + when: manual + + delayed-job: + script: "echo See you later, World!" + rules: + - if: $CI_COMMIT_REF_NAME =~ /master/ + when: delayed + start_in: 1 hour + + never-job: + script: "echo Goodbye, World!" + rules: + - if: $CI_COMMIT_REF_NAME + when: never + EOY + end + + context 'with matches' do + it 'creates a pipeline with the vanilla and manual jobs' do + expect(pipeline).to be_persisted + expect(build_names).to contain_exactly('regular-job', 'delayed-job', 'master-job') + end + + it 'assigns job:when values to the builds' do + expect(pipeline.builds.pluck(:when)).to contain_exactly('on_success', 'delayed', 'manual') + end + + it 'assigns start_in for delayed jobs' do + expect(delayed_job.options[:start_in]).to eq('1 hour') + end + end + + context 'with no matches' do + let(:ref_name) { 'refs/heads/feature' } + + it_behaves_like 'rules jobs are excluded' + end + end + + context 'with complex if: clauses' do + let(:config) do + <<-EOY + regular-job: + script: 'echo Hello, World!' + rules: + - if: $VAR == 'present' && $OTHER || $CI_COMMIT_REF_NAME + when: manual + EOY + end + + it 'matches the first rule' do + expect(pipeline).to be_persisted + expect(build_names).to contain_exactly('regular-job') + expect(regular_job.when).to eq('manual') + end + end + + context 'with changes:' do + let(:config) do + <<-EOY + regular-job: + script: 'echo Hello, World!' + + rules-job: + script: "echo hello world, $CI_COMMIT_REF_NAME" + rules: + - changes: + - README.md + when: manual + - changes: + - app.rb + when: on_success + + delayed-job: + script: "echo See you later, World!" + rules: + - changes: + - README.md + when: delayed + start_in: 4 hours + EOY + end + + context 'and matches' do + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[README.md]) + end + + it 'creates two jobs' do + expect(pipeline).to be_persisted + expect(build_names) + .to contain_exactly('regular-job', 'rules-job', 'delayed-job') + end + + it 'sets when: for all jobs' do + expect(regular_job.when).to eq('on_success') + expect(rules_job.when).to eq('manual') + expect(delayed_job.when).to eq('delayed') + expect(delayed_job.options[:start_in]).to eq('4 hours') + end + end + + context 'and matches the second rule' do + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[app.rb]) + end + + it 'includes both jobs' do + expect(pipeline).to be_persisted + expect(build_names).to contain_exactly('regular-job', 'rules-job') + end + + it 'sets when: for the created rules job based on the second clause' do + expect(regular_job.when).to eq('on_success') + expect(rules_job.when).to eq('on_success') + end + end + + context 'and does not match' do + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[useless_script.rb]) + end + + it_behaves_like 'rules jobs are excluded' + + it 'sets when: for the created job' do + expect(regular_job.when).to eq('on_success') + end + end + end + + context 'with mixed if: and changes: rules' do + let(:config) do + <<-EOY + regular-job: + script: 'echo Hello, World!' + + rules-job: + script: "echo hello world, $CI_COMMIT_REF_NAME" + rules: + - changes: + - README.md + when: manual + - if: $CI_COMMIT_REF_NAME == "master" + when: on_success + + delayed-job: + script: "echo See you later, World!" + rules: + - changes: + - README.md + when: delayed + start_in: 4 hours + - if: $CI_COMMIT_REF_NAME == "master" + when: delayed + start_in: 1 hour + EOY + end + + context 'and changes: matches before if' do + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[README.md]) + end + + it 'creates two jobs' do + expect(pipeline).to be_persisted + expect(build_names) + .to contain_exactly('regular-job', 'rules-job', 'delayed-job') + end + + it 'sets when: for all jobs' do + expect(regular_job.when).to eq('on_success') + expect(rules_job.when).to eq('manual') + expect(delayed_job.when).to eq('delayed') + expect(delayed_job.options[:start_in]).to eq('4 hours') + end + end + + context 'and if: matches after changes' do + it 'includes both jobs' do + expect(pipeline).to be_persisted + expect(build_names).to contain_exactly('regular-job', 'rules-job', 'delayed-job') + end + + it 'sets when: for the created rules job based on the second clause' do + expect(regular_job.when).to eq('on_success') + expect(rules_job.when).to eq('on_success') + expect(delayed_job.when).to eq('delayed') + expect(delayed_job.options[:start_in]).to eq('1 hour') + end + end + + context 'and does not match' do + let(:ref_name) { 'refs/heads/wip' } + + it_behaves_like 'rules jobs are excluded' + + it 'sets when: for the created job' do + expect(regular_job.when).to eq('on_success') + end + end + end + + context 'with mixed if: and changes: clauses' do + let(:config) do + <<-EOY + regular-job: + script: 'echo Hello, World!' + + rules-job: + script: "echo hello world, $CI_COMMIT_REF_NAME" + rules: + - if: $CI_COMMIT_REF_NAME =~ /master/ + changes: [README.md] + when: on_success + - if: $CI_COMMIT_REF_NAME =~ /master/ + changes: [app.rb] + when: manual + EOY + end + + context 'with if matches and changes matches' do + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[app.rb]) + end + + it 'persists all jobs' do + expect(pipeline).to be_persisted + expect(regular_job).to be_persisted + expect(rules_job).to be_persisted + expect(rules_job.when).to eq('manual') + end + end + + context 'with if matches and no change matches' do + it_behaves_like 'rules jobs are excluded' + end + + context 'with change matches and no if matches' do + let(:ref_name) { 'refs/heads/feature' } + + before do + allow_any_instance_of(Ci::Pipeline) + .to receive(:modified_paths).and_return(%w[README.md]) + end + + it_behaves_like 'rules jobs are excluded' + end + + context 'and no matches' do + let(:ref_name) { 'refs/heads/feature' } + + it_behaves_like 'rules jobs are excluded' end end end diff --git a/spec/support/import_export/import_export.yml b/spec/support/import_export/import_export.yml index 734d6838f4d..ed2a3243f0d 100644 --- a/spec/support/import_export/import_export.yml +++ b/spec/support/import_export/import_export.yml @@ -1,13 +1,16 @@ # Class relationships to be included in the project import/export -project_tree: - - :issues - - :labels - - merge_requests: - - :merge_request_diff - - :merge_request_test - - commit_statuses: - - :commit - - project_members: +tree: + project: + - :issues + - :labels + - merge_requests: + - :merge_request_diff + - :merge_request_test + - commit_statuses: + - :commit + - project_members: + - :user + group_members: - :user included_attributes: |
