diff options
534 files changed, 9360 insertions, 4417 deletions
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 2d06a8acc58..c7a51beeee5 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -66,6 +66,8 @@ - scripts/gitaly-test-spawn - date - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")' + - mkdir -p tmp/memory_test + - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo" - date artifacts: @@ -77,6 +79,7 @@ - rspec_flaky/ - rspec_profiling/ - tmp/capybara/ + - tmp/memory_test/ # reports: # junit: junit_rspec.xml @@ -273,6 +276,7 @@ coverage: stage: post-test script: - bundle exec scripts/merge-simplecov + - bundle exec scripts/gather-test-memory-data coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage @@ -280,3 +284,4 @@ coverage: paths: - coverage/index.html - coverage/assets/ + - tmp/memory_test/ diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 89b5ae38072..ca55bbd32a7 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -10,17 +10,17 @@ code_quality: tags: [] before_script: [] cache: {} - dependencies: [] sast: extends: .dedicated-no-docs - before_script: [] tags: [] + before_script: [] + cache: {} variables: - SAST_CONFIDENCE_LEVEL: 2 - DOCKER_DRIVER: overlay2 + SAST_BRAKEMAN_LEVEL: 2 dependency_scanning: - before_script: [] - tags: [] extends: .dedicated-no-docs + tags: [] + before_script: [] + cache: {} diff --git a/.gitlab/issue_templates/Documentation.md b/.gitlab/issue_templates/Documentation.md index 67602b7b2df..f4070a44755 100644 --- a/.gitlab/issue_templates/Documentation.md +++ b/.gitlab/issue_templates/Documentation.md @@ -4,7 +4,7 @@ Note: Doc work as part of feature development is covered in the Feature Request template. * For issues related to features of the docs.gitlab.com site, see - https://gitlab.com/gitlab-com/gitlab-docs/issues/ + https://gitlab.com/gitlab-org/gitlab-docs/issues/ * For information about documentation content and process, see https://docs.gitlab.com/ee/development/documentation/ --> diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index a63cb35e6f0..3f4830156cb 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -1.52.0 +1.53.0 diff --git a/GITLAB_PAGES_VERSION b/GITLAB_PAGES_VERSION index 9c6d6293b1a..bd8bf882d06 100644 --- a/GITLAB_PAGES_VERSION +++ b/GITLAB_PAGES_VERSION @@ -1 +1 @@ -1.6.1 +1.7.0 @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'rails', '5.1.7' +gem 'rails', '5.2.3' # Improves copy-on-write performance for MRI gem 'nakayoshi_fork', '~> 0.0.4' @@ -84,6 +84,7 @@ gem 'rack-cors', '~> 1.0.0', require: 'rack/cors' gem 'graphql', '~> 1.8.0' gem 'graphiql-rails', '~> 1.4.10' gem 'apollo_upload_server', '~> 2.0.0.beta3' +gem 'graphql-docs', '~> 1.6.0', group: [:development, :test] # Disable strong_params so that Mash does not respond to :permitted? gem 'hashie-forbidden_attributes' @@ -339,7 +340,7 @@ group :development, :test do gem 'database_cleaner', '~> 1.7.0' gem 'factory_bot_rails', '~> 4.8.2' - gem 'rspec-rails', '~> 3.7.0' + gem 'rspec-rails', '~> 3.8.0' gem 'rspec-retry', '~> 0.6.1' gem 'rspec_profiling', '~> 0.0.5' gem 'rspec-set', '~> 0.1.3' diff --git a/Gemfile.lock b/Gemfile.lock index 85b4c32f168..bf469de3835 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,44 +6,48 @@ GEM ace-rails-ap (4.1.2) acme-client (2.0.2) faraday (~> 0.9, >= 0.9.1) - actioncable (5.1.7) - actionpack (= 5.1.7) + actioncable (5.2.3) + actionpack (= 5.2.3) nio4r (~> 2.0) - websocket-driver (~> 0.6.1) - actionmailer (5.1.7) - actionpack (= 5.1.7) - actionview (= 5.1.7) - activejob (= 5.1.7) + websocket-driver (>= 0.6.1) + actionmailer (5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.1.7) - actionview (= 5.1.7) - activesupport (= 5.1.7) + actionpack (5.2.3) + actionview (= 5.2.3) + activesupport (= 5.2.3) rack (~> 2.0) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.1.7) - activesupport (= 5.1.7) + actionview (5.2.3) + activesupport (= 5.2.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.1.7) - activesupport (= 5.1.7) + activejob (5.2.3) + activesupport (= 5.2.3) globalid (>= 0.3.6) - activemodel (5.1.7) - activesupport (= 5.1.7) - activerecord (5.1.7) - activemodel (= 5.1.7) - activesupport (= 5.1.7) - arel (~> 8.0) + activemodel (5.2.3) + activesupport (= 5.2.3) + activerecord (5.2.3) + activemodel (= 5.2.3) + activesupport (= 5.2.3) + arel (>= 9.0) activerecord-explain-analyze (0.1.0) activerecord (>= 4) pg activerecord_sane_schema_dumper (1.0) rails (>= 5, < 6) - activesupport (5.1.7) + activestorage (5.2.3) + actionpack (= 5.2.3) + activerecord (= 5.2.3) + marcel (~> 0.3.1) + activesupport (5.2.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -60,7 +64,7 @@ GEM apollo_upload_server (2.0.0.beta.3) graphql (>= 1.8) rails (>= 4.2) - arel (8.0.0) + arel (9.0.0) asana (0.8.1) faraday (~> 0.9) faraday_middleware (~> 0.9) @@ -216,6 +220,8 @@ GEM excon (0.62.0) execjs (2.6.0) expression_parser (0.9.0) + extended-markdown-filter (0.6.0) + html-pipeline (~> 2.0) factory_bot (4.8.2) activesupport (>= 3.0.0) factory_bot_rails (4.8.2) @@ -290,6 +296,7 @@ GEM fuubar (2.2.0) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + gemoji (3.0.1) gemojione (3.3.0) json get_process_mem (0.2.3) @@ -370,6 +377,14 @@ GEM railties sprockets-rails graphql (1.8.1) + graphql-docs (1.6.0) + commonmarker (~> 0.16) + escape_utils (~> 1.2) + extended-markdown-filter (~> 0.4) + gemoji (~> 3.0) + graphql (~> 1.6) + html-pipeline (~> 2.8) + sass (~> 3.4) grpc (1.19.0) google-protobuf (~> 3.1) googleapis-common-protos-types (~> 1.0.0) @@ -493,6 +508,8 @@ GEM mail (2.7.1) mini_mime (>= 0.1.1) mail_room (0.9.1) + marcel (0.3.3) + mimemagic (~> 0.3.2) mdl (0.5.0) kramdown (~> 1.12, >= 1.12.0) mixlib-cli (~> 1.7, >= 1.7.0) @@ -695,17 +712,18 @@ GEM rack-test (1.1.0) rack (>= 1.0, < 3) rack-timeout (0.5.1) - rails (5.1.7) - actioncable (= 5.1.7) - actionmailer (= 5.1.7) - actionpack (= 5.1.7) - actionview (= 5.1.7) - activejob (= 5.1.7) - activemodel (= 5.1.7) - activerecord (= 5.1.7) - activesupport (= 5.1.7) + rails (5.2.3) + actioncable (= 5.2.3) + actionmailer (= 5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) + activemodel (= 5.2.3) + activerecord (= 5.2.3) + activestorage (= 5.2.3) + activesupport (= 5.2.3) bundler (>= 1.3.0) - railties (= 5.1.7) + railties (= 5.2.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.2) actionpack (~> 5.x, >= 5.0.1) @@ -719,12 +737,12 @@ GEM rails-i18n (5.1.1) i18n (>= 0.7, < 2) railties (>= 5.0, < 6) - railties (5.1.7) - actionpack (= 5.1.7) - activesupport (= 5.1.7) + railties (5.2.3) + actionpack (= 5.2.3) + activesupport (= 5.2.3) method_source rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) + thor (>= 0.19.0, < 2.0) rainbow (3.0.0) raindrops (0.19.0) rake (12.3.2) @@ -783,36 +801,36 @@ GEM chunky_png rqrcode-rails3 (0.1.7) rqrcode (>= 0.4.2) - rspec (3.7.0) - rspec-core (~> 3.7.0) - rspec-expectations (~> 3.7.0) - rspec-mocks (~> 3.7.0) - rspec-core (3.7.1) - rspec-support (~> 3.7.0) - rspec-expectations (3.7.0) + rspec (3.8.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-core (3.8.2) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.4) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-mocks (3.7.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.1) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) + rspec-support (~> 3.8.0) rspec-parameterized (0.4.2) binding_ninja (>= 0.2.3) parser proc_to_ast rspec (>= 2.13, < 4) unparser - rspec-rails (3.7.2) + rspec-rails (3.8.2) actionpack (>= 3.0) activesupport (>= 3.0) railties (>= 3.0) - rspec-core (~> 3.7.0) - rspec-expectations (~> 3.7.0) - rspec-mocks (~> 3.7.0) - rspec-support (~> 3.7.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) rspec-retry (0.6.1) rspec-core (> 3.3) rspec-set (0.1.3) - rspec-support (3.7.1) + rspec-support (3.8.2) rspec_junit_formatter (0.4.1) rspec-core (>= 2, < 4, != 2.12.0) rspec_profiling (0.0.5) @@ -1008,7 +1026,7 @@ GEM hashdiff webpack-rails (0.9.11) railties (>= 3.2.0) - websocket-driver (0.6.5) + websocket-driver (0.7.0) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) wikicloth (0.8.1) @@ -1118,6 +1136,7 @@ DEPENDENCIES grape_logging (~> 1.7) graphiql-rails (~> 1.4.10) graphql (~> 1.8.0) + graphql-docs (~> 1.6.0) grpc (~> 1.19.0) haml_lint (~> 0.31.0) hamlit (~> 2.8.8) @@ -1194,7 +1213,7 @@ DEPENDENCIES rack-oauth2 (~> 1.9.3) rack-proxy (~> 0.6.0) rack-timeout - rails (= 5.1.7) + rails (= 5.2.3) rails-controller-testing rails-i18n (~> 5.1) rainbow (~> 3.0) @@ -1212,7 +1231,7 @@ DEPENDENCIES rouge (~> 3.5) rqrcode-rails3 (~> 0.1.7) rspec-parameterized - rspec-rails (~> 3.7.0) + rspec-rails (~> 3.8.0) rspec-retry (~> 0.6.1) rspec-set (~> 0.1.3) rspec_junit_formatter diff --git a/app/assets/javascripts/boards/components/modal/footer.vue b/app/assets/javascripts/boards/components/modal/footer.vue index a1d634c8f19..eea0fb71c1a 100644 --- a/app/assets/javascripts/boards/components/modal/footer.vue +++ b/app/assets/javascripts/boards/components/modal/footer.vue @@ -41,7 +41,7 @@ export default { const req = this.buildUpdateRequest(list); // Post the data to the backend - gl.boardService.bulkUpdate(issueIds, req).catch(() => { + boardsStore.bulkUpdate(issueIds, req).catch(() => { Flash(__('Failed to update issues, please try again.')); selectedIssues.forEach(issue => { diff --git a/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js b/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js index b56e08175cc..d4b994d4922 100644 --- a/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js +++ b/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js @@ -17,6 +17,7 @@ Vue.use(Translate); export default () => { const OVERVIEW_DIALOG_COOKIE = 'cycle_analytics_help_dismissed'; + const cycleAnalyticsEl = document.querySelector('#cycle-analytics'); // eslint-disable-next-line no-new new Vue({ @@ -33,7 +34,6 @@ export default () => { 'stage-production-component': stageComponent, }, data() { - const cycleAnalyticsEl = document.querySelector('#cycle-analytics'); const cycleAnalyticsService = new CycleAnalyticsService({ requestPath: cycleAnalyticsEl.dataset.requestPath, }); @@ -56,7 +56,13 @@ export default () => { }, }, created() { - this.fetchCycleAnalyticsData(); + // Conditional check placed here to prevent this method from being called on the + // new Cycle Analytics page (i.e. the new page will be initialized blank and only + // after a group is selected the cycle analyitcs data will be fetched). Once the + // old (current) page has been removed this entire created method as well as the + // variable itself can be completely removed. + // Follow up issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/64490 + if (cycleAnalyticsEl.dataset.requestPath) this.fetchCycleAnalyticsData(); }, methods: { handleError() { diff --git a/app/assets/javascripts/filterable_list.js b/app/assets/javascripts/filterable_list.js index 64b09c8b62c..77080691dcb 100644 --- a/app/assets/javascripts/filterable_list.js +++ b/app/assets/javascripts/filterable_list.js @@ -17,11 +17,13 @@ export default class FilterableList { } getFilterEndpoint() { - return `${this.filterForm.getAttribute('action')}?${$(this.filterForm).serialize()}`; + return this.getPagePath(); } getPagePath() { - return this.getFilterEndpoint(); + const action = this.filterForm.getAttribute('action'); + const params = $(this.filterForm).serialize(); + return `${action}${action.indexOf('?') > 0 ? '&' : '?'}${params}`; } initSearch() { diff --git a/app/assets/javascripts/monitoring/components/charts/area.vue b/app/assets/javascripts/monitoring/components/charts/area.vue index 81773bd140e..454ff4f284e 100644 --- a/app/assets/javascripts/monitoring/components/charts/area.vue +++ b/app/assets/javascripts/monitoring/components/charts/area.vue @@ -8,6 +8,7 @@ import { getSvgIconPathContent } from '~/lib/utils/icon_utils'; import Icon from '~/vue_shared/components/icon.vue'; import { chartHeight, graphTypes, lineTypes } from '../../constants'; import { makeDataSeries } from '~/helpers/monitor_helper'; +import { graphDataValidatorForValues } from '../../utils'; let debouncedResize; @@ -23,19 +24,7 @@ export default { graphData: { type: Object, required: true, - validator(data) { - return ( - Array.isArray(data.queries) && - data.queries.filter(query => { - if (Array.isArray(query.result)) { - return ( - query.result.filter(res => Array.isArray(res.values)).length === query.result.length - ); - } - return false; - }).length === data.queries.length - ); - }, + validator: graphDataValidatorForValues.bind(null, false), }, containerWidth: { type: Number, diff --git a/app/assets/javascripts/monitoring/components/charts/column.vue b/app/assets/javascripts/monitoring/components/charts/column.vue index 05a2036f4c3..83136d43479 100644 --- a/app/assets/javascripts/monitoring/components/charts/column.vue +++ b/app/assets/javascripts/monitoring/components/charts/column.vue @@ -4,6 +4,7 @@ import { debounceByAnimationFrame } from '~/lib/utils/common_utils'; import { getSvgIconPathContent } from '~/lib/utils/icon_utils'; import { chartHeight } from '../../constants'; import { makeDataSeries } from '~/helpers/monitor_helper'; +import { graphDataValidatorForValues } from '../../utils'; export default { components: { @@ -14,23 +15,11 @@ export default { graphData: { type: Object, required: true, - validator(data) { - return ( - Array.isArray(data.queries) && - data.queries.filter(query => { - if (Array.isArray(query.result)) { - return ( - query.result.filter(res => Array.isArray(res.values)).length === query.result.length - ); - } - return false; - }).length === data.queries.length - ); - }, - containerWidth: { - type: Number, - required: true, - }, + validator: graphDataValidatorForValues.bind(null, false), + }, + containerWidth: { + type: Number, + required: true, }, }, data() { diff --git a/app/assets/javascripts/monitoring/components/charts/single_stat.vue b/app/assets/javascripts/monitoring/components/charts/single_stat.vue index b03a6ca1806..7428b27a9c3 100644 --- a/app/assets/javascripts/monitoring/components/charts/single_stat.vue +++ b/app/assets/javascripts/monitoring/components/charts/single_stat.vue @@ -1,5 +1,7 @@ <script> import { GlSingleStat } from '@gitlab/ui/dist/charts'; +import { roundOffFloat } from '~/lib/utils/common_utils'; +import { graphDataValidatorForValues } from '../../utils'; export default { components: { @@ -7,22 +9,21 @@ export default { }, inheritAttrs: false, props: { - title: { - type: String, - required: true, - }, - value: { - type: Number, - required: true, - }, - unit: { - type: String, + graphData: { + type: Object, required: true, + validator: graphDataValidatorForValues.bind(null, true), }, }, computed: { - valueWithUnit() { - return `${this.value}${this.unit}`; + queryInfo() { + return this.graphData.queries[0]; + }, + engineeringNotation() { + return `${roundOffFloat(this.queryInfo.result[0].value[1], 1)}${this.queryInfo.unit}`; + }, + graphTitle() { + return this.queryInfo.label; }, }, }; @@ -30,8 +31,8 @@ export default { <template> <div class="prometheus-graph col-12 col-lg-6"> <div class="prometheus-graph-header"> - <h5 ref="graphTitle" class="prometheus-graph-title">{{ title }}</h5> + <h5 ref="graphTitle" class="prometheus-graph-title">{{ graphTitle }}</h5> </div> - <gl-single-stat :value="valueWithUnit" :title="title" variant="success" /> + <gl-single-stat :value="engineeringNotation" :title="graphTitle" variant="success" /> </div> </template> diff --git a/app/assets/javascripts/monitoring/components/dashboard.vue b/app/assets/javascripts/monitoring/components/dashboard.vue index ba79a697df2..6eaced0c108 100644 --- a/app/assets/javascripts/monitoring/components/dashboard.vue +++ b/app/assets/javascripts/monitoring/components/dashboard.vue @@ -7,6 +7,8 @@ import Icon from '~/vue_shared/components/icon.vue'; import { getParameterValues } from '~/lib/utils/url_utility'; import invalidUrl from '~/lib/utils/invalid_url'; import MonitorAreaChart from './charts/area.vue'; +import MonitorSingleStatChart from './charts/single_stat.vue'; +import PanelType from './panel_type.vue'; import GraphGroup from './graph_group.vue'; import EmptyState from './empty_state.vue'; import { timeWindows, timeWindowsKeyNames } from '../constants'; @@ -18,6 +20,8 @@ let sidebarMutationObserver; export default { components: { MonitorAreaChart, + MonitorSingleStatChart, + PanelType, GraphGroup, EmptyState, Icon, @@ -152,6 +156,7 @@ export default { 'useDashboardEndpoint', 'allDashboards', 'multipleDashboardsEnabled', + 'additionalPanelTypesEnabled', ]), groupsWithData() { return this.groups.filter(group => this.chartsWithData(group.metrics).length > 0); @@ -173,6 +178,7 @@ export default { deploymentsEndpoint: this.deploymentsEndpoint, dashboardEndpoint: this.dashboardEndpoint, currentDashboard: this.currentDashboard, + projectPath: this.projectPath, }); this.timeWindows = timeWindows; @@ -220,6 +226,8 @@ export default { chart.metrics.some(metric => this.metricsWithData.includes(metric.metric_id)), ); }, + // TODO: BEGIN, Duplicated code with panel_type until feature flag is removed + // Issue number: https://gitlab.com/gitlab-org/gitlab-ce/issues/63845 getGraphAlerts(queries) { if (!this.allAlerts) return {}; const metricIdsForChart = queries.map(q => q.metricId); @@ -228,6 +236,7 @@ export default { getGraphAlertValues(queries) { return Object.values(this.getGraphAlerts(queries)); }, + // TODO: END hideAddMetricModal() { this.$refs.addMetricModal.hide(); }, @@ -366,24 +375,34 @@ export default { :name="groupData.group" :show-panels="showPanels" > - <monitor-area-chart - v-for="(graphData, graphIndex) in chartsWithData(groupData.metrics)" - :key="graphIndex" - :project-path="projectPath" - :graph-data="graphData" - :deployment-data="deploymentData" - :thresholds="getGraphAlertValues(graphData.queries)" - :container-width="elWidth" - group-id="monitor-area-chart" - > - <alert-widget - v-if="alertWidgetAvailable && graphData" - :alerts-endpoint="alertsEndpoint" - :relevant-queries="graphData.queries" - :alerts-to-manage="getGraphAlerts(graphData.queries)" - @setAlerts="setAlerts" + <template v-if="additionalPanelTypesEnabled"> + <panel-type + v-for="(graphData, graphIndex) in chartsWithData(groupData.metrics)" + :key="`panel-type-${graphIndex}`" + :graph-data="graphData" + :dashboard-width="elWidth" /> - </monitor-area-chart> + </template> + <template v-else> + <monitor-area-chart + v-for="(graphData, graphIndex) in chartsWithData(groupData.metrics)" + :key="graphIndex" + :graph-data="graphData" + :deployment-data="deploymentData" + :thresholds="getGraphAlertValues(graphData.queries)" + :container-width="elWidth" + :project-path="projectPath" + group-id="monitor-area-chart" + > + <alert-widget + v-if="alertWidgetAvailable && graphData" + :alerts-endpoint="alertsEndpoint" + :relevant-queries="graphData.queries" + :alerts-to-manage="getGraphAlerts(graphData.queries)" + @setAlerts="setAlerts" + /> + </monitor-area-chart> + </template> </graph-group> </div> <empty-state diff --git a/app/assets/javascripts/monitoring/components/panel_type.vue b/app/assets/javascripts/monitoring/components/panel_type.vue new file mode 100644 index 00000000000..45ee067de83 --- /dev/null +++ b/app/assets/javascripts/monitoring/components/panel_type.vue @@ -0,0 +1,62 @@ +<script> +import { mapState } from 'vuex'; +import _ from 'underscore'; +import MonitorAreaChart from './charts/area.vue'; +import MonitorSingleStatChart from './charts/single_stat.vue'; + +export default { + components: { + MonitorAreaChart, + MonitorSingleStatChart, + }, + props: { + graphData: { + type: Object, + required: true, + }, + dashboardWidth: { + type: Number, + required: true, + }, + }, + computed: { + ...mapState('monitoringDashboard', ['deploymentData', 'projectPath']), + alertWidgetAvailable() { + return IS_EE && this.prometheusAlertsAvailable && this.alertsEndpoint && this.graphData; + }, + }, + methods: { + getGraphAlerts(queries) { + if (!this.allAlerts) return {}; + const metricIdsForChart = queries.map(q => q.metricId); + return _.pick(this.allAlerts, alert => metricIdsForChart.includes(alert.metricId)); + }, + getGraphAlertValues(queries) { + return Object.values(this.getGraphAlerts(queries)); + }, + isPanelType(type) { + return this.graphData.type && this.graphData.type === type; + }, + }, +}; +</script> +<template> + <monitor-single-stat-chart v-if="isPanelType('single-stat')" :graph-data="graphData" /> + <monitor-area-chart + v-else + :graph-data="graphData" + :deployment-data="deploymentData" + :project-path="projectPath" + :thresholds="getGraphAlertValues(graphData.queries)" + :container-width="dashboardWidth" + group-id="monitor-area-chart" + > + <alert-widget + v-if="alertWidgetAvailable" + :alerts-endpoint="alertsEndpoint" + :relevant-queries="graphData.queries" + :alerts-to-manage="getGraphAlerts(graphData.queries)" + @setAlerts="setAlerts" + /> + </monitor-area-chart> +</template> diff --git a/app/assets/javascripts/monitoring/monitoring_bundle.js b/app/assets/javascripts/monitoring/monitoring_bundle.js index 97d149e9ad5..c0fee1ebb99 100644 --- a/app/assets/javascripts/monitoring/monitoring_bundle.js +++ b/app/assets/javascripts/monitoring/monitoring_bundle.js @@ -12,6 +12,7 @@ export default (props = {}) => { store.dispatch('monitoringDashboard/setFeatureFlags', { prometheusEndpointEnabled: gon.features.environmentMetricsUsePrometheusEndpoint, multipleDashboardsEnabled: gon.features.environmentMetricsShowMultipleDashboards, + additionalPanelTypesEnabled: gon.features.environmentMetricsAdditionalPanelTypes, }); } diff --git a/app/assets/javascripts/monitoring/stores/actions.js b/app/assets/javascripts/monitoring/stores/actions.js index 0fa2a5d6370..5b3da51e9a6 100644 --- a/app/assets/javascripts/monitoring/stores/actions.js +++ b/app/assets/javascripts/monitoring/stores/actions.js @@ -37,10 +37,11 @@ export const setEndpoints = ({ commit }, endpoints) => { export const setFeatureFlags = ( { commit }, - { prometheusEndpointEnabled, multipleDashboardsEnabled }, + { prometheusEndpointEnabled, multipleDashboardsEnabled, additionalPanelTypesEnabled }, ) => { commit(types.SET_DASHBOARD_ENABLED, prometheusEndpointEnabled); commit(types.SET_MULTIPLE_DASHBOARDS_ENABLED, multipleDashboardsEnabled); + commit(types.SET_ADDITIONAL_PANEL_TYPES_ENABLED, additionalPanelTypesEnabled); }; export const requestMetricsDashboard = ({ commit }) => { diff --git a/app/assets/javascripts/monitoring/stores/mutation_types.js b/app/assets/javascripts/monitoring/stores/mutation_types.js index 2c78a0b9315..c89daba3df7 100644 --- a/app/assets/javascripts/monitoring/stores/mutation_types.js +++ b/app/assets/javascripts/monitoring/stores/mutation_types.js @@ -11,6 +11,7 @@ export const SET_QUERY_RESULT = 'SET_QUERY_RESULT'; export const SET_TIME_WINDOW = 'SET_TIME_WINDOW'; export const SET_DASHBOARD_ENABLED = 'SET_DASHBOARD_ENABLED'; export const SET_MULTIPLE_DASHBOARDS_ENABLED = 'SET_MULTIPLE_DASHBOARDS_ENABLED'; +export const SET_ADDITIONAL_PANEL_TYPES_ENABLED = 'SET_ADDITIONAL_PANEL_TYPES_ENABLED'; export const SET_ALL_DASHBOARDS = 'SET_ALL_DASHBOARDS'; export const SET_ENDPOINTS = 'SET_ENDPOINTS'; export const SET_GETTING_STARTED_EMPTY_STATE = 'SET_GETTING_STARTED_EMPTY_STATE'; diff --git a/app/assets/javascripts/monitoring/stores/mutations.js b/app/assets/javascripts/monitoring/stores/mutations.js index a85a7723c1f..0104dcb867d 100644 --- a/app/assets/javascripts/monitoring/stores/mutations.js +++ b/app/assets/javascripts/monitoring/stores/mutations.js @@ -75,6 +75,7 @@ export default { state.deploymentsEndpoint = endpoints.deploymentsEndpoint; state.dashboardEndpoint = endpoints.dashboardEndpoint; state.currentDashboard = endpoints.currentDashboard; + state.projectPath = endpoints.projectPath; }, [types.SET_DASHBOARD_ENABLED](state, enabled) { state.useDashboardEndpoint = enabled; @@ -92,4 +93,7 @@ export default { [types.SET_ALL_DASHBOARDS](state, dashboards) { state.allDashboards = dashboards; }, + [types.SET_ADDITIONAL_PANEL_TYPES_ENABLED](state, enabled) { + state.additionalPanelTypesEnabled = enabled; + }, }; diff --git a/app/assets/javascripts/monitoring/stores/state.js b/app/assets/javascripts/monitoring/stores/state.js index de711d6ccae..e54bb712695 100644 --- a/app/assets/javascripts/monitoring/stores/state.js +++ b/app/assets/javascripts/monitoring/stores/state.js @@ -9,6 +9,7 @@ export default () => ({ dashboardEndpoint: invalidUrl, useDashboardEndpoint: false, multipleDashboardsEnabled: false, + additionalPanelTypesEnabled: false, emptyState: 'gettingStarted', showEmptyState: true, groups: [], @@ -17,4 +18,5 @@ export default () => ({ metricsWithData: [], allDashboards: [], currentDashboard: null, + projectPath: null, }); diff --git a/app/assets/javascripts/monitoring/stores/utils.js b/app/assets/javascripts/monitoring/stores/utils.js index 721942f9d3b..938ee2f0a9a 100644 --- a/app/assets/javascripts/monitoring/stores/utils.js +++ b/app/assets/javascripts/monitoring/stores/utils.js @@ -69,13 +69,26 @@ export const sortMetrics = metrics => .sortBy('weight') .value(); -export const normalizeQueryResult = timeSeries => ({ - ...timeSeries, - values: timeSeries.values.map(([timestamp, value]) => [ - new Date(timestamp * 1000).toISOString(), - Number(value), - ]), -}); +export const normalizeQueryResult = timeSeries => { + let normalizedResult = {}; + + if (timeSeries.values) { + normalizedResult = { + ...timeSeries, + values: timeSeries.values.map(([timestamp, value]) => [ + new Date(timestamp * 1000).toISOString(), + Number(value), + ]), + }; + } else if (timeSeries.value) { + normalizedResult = { + ...timeSeries, + value: [new Date(timeSeries.value[0] * 1000).toISOString(), Number(timeSeries.value[1])], + }; + } + + return normalizedResult; +}; export const normalizeMetrics = metrics => { const groupedMetrics = groupQueriesByChartInfo(metrics); diff --git a/app/assets/javascripts/monitoring/utils.js b/app/assets/javascripts/monitoring/utils.js index ef309c8a398..478e2b3d06c 100644 --- a/app/assets/javascripts/monitoring/utils.js +++ b/app/assets/javascripts/monitoring/utils.js @@ -30,4 +30,28 @@ export const getTimeDiff = selectedTimeWindow => { return { start, end }; }; +/** + * This method is used to validate if the graph data format for a chart component + * that needs a time series as a response from a prometheus query (query_range) is + * of a valid format or not. + * @param {Object} graphData the graph data response from a prometheus request + * @returns {boolean} whether the graphData format is correct + */ +export const graphDataValidatorForValues = (isValues, graphData) => { + const responseValueKeyName = isValues ? 'value' : 'values'; + + return ( + Array.isArray(graphData.queries) && + graphData.queries.filter(query => { + if (Array.isArray(query.result)) { + return ( + query.result.filter(res => Array.isArray(res[responseValueKeyName])).length === + query.result.length + ); + } + return false; + }).length === graphData.queries.length + ); +}; + export default {}; diff --git a/app/assets/javascripts/notes/components/note_actions.vue b/app/assets/javascripts/notes/components/note_actions.vue index 844d0c3e376..6cc873359da 100644 --- a/app/assets/javascripts/notes/components/note_actions.vue +++ b/app/assets/javascripts/notes/components/note_actions.vue @@ -165,7 +165,7 @@ export default { v-gl-tooltip type="button" title="Edit comment" - class="note-action-button js-note-edit btn btn-transparent" + class="note-action-button js-note-edit btn btn-transparent qa-note-edit-button" @click="onEdit" > <icon name="pencil" css-classes="link-highlight" /> diff --git a/app/assets/javascripts/notes/services/notes_service.js b/app/assets/javascripts/notes/services/notes_service.js index bc0f5c19b9d..9e0392110b6 100644 --- a/app/assets/javascripts/notes/services/notes_service.js +++ b/app/assets/javascripts/notes/services/notes_service.js @@ -9,9 +9,6 @@ export default { const config = filter !== undefined ? { params: { notes_filter: filter } } : null; return Vue.http.get(endpoint, config); }, - deleteNote(endpoint) { - return Vue.http.delete(endpoint); - }, replyToDiscussion(endpoint, data) { return Vue.http.post(endpoint, data, { emulateJSON: true }); }, diff --git a/app/assets/javascripts/notes/stores/actions.js b/app/assets/javascripts/notes/stores/actions.js index 2eefef8bd6e..30eab272aa9 100644 --- a/app/assets/javascripts/notes/stores/actions.js +++ b/app/assets/javascripts/notes/stores/actions.js @@ -62,7 +62,7 @@ export const updateDiscussion = ({ commit, state }, discussion) => { }; export const deleteNote = ({ commit, dispatch, state }, note) => - service.deleteNote(note.path).then(() => { + axios.delete(note.path).then(() => { const discussion = state.discussions.find(({ id }) => id === note.discussion_id); commit(types.DELETE_NOTE, note); diff --git a/app/assets/javascripts/projects/projects_filterable_list.js b/app/assets/javascripts/projects/projects_filterable_list.js new file mode 100644 index 00000000000..433c894e668 --- /dev/null +++ b/app/assets/javascripts/projects/projects_filterable_list.js @@ -0,0 +1,7 @@ +import FilterableList from '~/filterable_list'; + +export default class ProjectsFilterableList extends FilterableList { + getFilterEndpoint() { + return this.getPagePath().replace('/projects?', '/projects.json?'); + } +} diff --git a/app/assets/javascripts/projects_list.js b/app/assets/javascripts/projects_list.js index c67d59d2be5..913b62ba26d 100644 --- a/app/assets/javascripts/projects_list.js +++ b/app/assets/javascripts/projects_list.js @@ -1,4 +1,4 @@ -import FilterableList from './filterable_list'; +import ProjectsFilterableList from './projects/projects_filterable_list'; /** * Makes search request for projects when user types a value in the search input. @@ -11,7 +11,7 @@ export default class ProjectsList { const holder = document.querySelector('.js-projects-list-holder'); if (form && filter && holder) { - const list = new FilterableList(form, filter, holder); + const list = new ProjectsFilterableList(form, filter, holder); list.initSearch(); } } diff --git a/app/assets/javascripts/registry/components/collapsible_container.vue b/app/assets/javascripts/registry/components/collapsible_container.vue index 1e266dd4ced..e157036871b 100644 --- a/app/assets/javascripts/registry/components/collapsible_container.vue +++ b/app/assets/javascripts/registry/components/collapsible_container.vue @@ -31,6 +31,7 @@ export default { data() { return { isOpen: false, + modalId: `confirm-repo-deletion-modal-${this.repo.id}`, }; }, computed: { @@ -80,7 +81,7 @@ export default { <gl-button v-if="repo.canDelete" v-gl-tooltip - v-gl-modal="'confirm-repo-deletion-modal'" + v-gl-modal="modalId" :title="s__('ContainerRegistry|Remove repository')" :aria-label="s__('ContainerRegistry|Remove repository')" class="js-remove-repo" @@ -100,12 +101,7 @@ export default { {{ s__('ContainerRegistry|No tags in Container Registry for this container image.') }} </div> </div> - - <gl-modal - modal-id="confirm-repo-deletion-modal" - ok-variant="danger" - @ok="handleDeleteRepository" - > + <gl-modal :modal-id="modalId" ok-variant="danger" @ok="handleDeleteRepository"> <template v-slot:modal-title>{{ s__('ContainerRegistry|Remove repository') }}</template> <p v-html=" diff --git a/app/assets/javascripts/registry/components/table_registry.vue b/app/assets/javascripts/registry/components/table_registry.vue index 0ec5e2c7a87..a498a553908 100644 --- a/app/assets/javascripts/registry/components/table_registry.vue +++ b/app/assets/javascripts/registry/components/table_registry.vue @@ -32,6 +32,7 @@ export default { data() { return { itemToBeDeleted: null, + modalId: `confirm-image-deletion-modal-${this.repo.id}`, }; }, computed: { @@ -114,7 +115,7 @@ export default { <gl-button v-if="item.canDelete" v-gl-tooltip - v-gl-modal="'confirm-image-deletion-modal'" + v-gl-modal="modalId" :title="s__('ContainerRegistry|Remove image')" :aria-label="s__('ContainerRegistry|Remove image')" variant="danger" @@ -134,11 +135,7 @@ export default { :page-info="repo.pagination" /> - <gl-modal - modal-id="confirm-image-deletion-modal" - ok-variant="danger" - @ok="handleDeleteRegistry" - > + <gl-modal :modal-id="modalId" ok-variant="danger" @ok="handleDeleteRegistry"> <template v-slot:modal-title>{{ s__('ContainerRegistry|Remove image') }}</template> <template v-slot:modal-ok>{{ s__('ContainerRegistry|Remove image and tags') }}</template> <p diff --git a/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css b/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css index 00a55c0027a..6a7b2f52549 100644 --- a/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css +++ b/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css @@ -48,6 +48,7 @@ font-size: .8rem; font-weight: 400; color: #2e2e2e; + z-index: 9999; /* toolbar should always be on top */ } .gitlab-wrapper-open { diff --git a/app/assets/stylesheets/framework.scss b/app/assets/stylesheets/framework.scss index 9b1d9d51f9c..82b4ec750ff 100644 --- a/app/assets/stylesheets/framework.scss +++ b/app/assets/stylesheets/framework.scss @@ -9,7 +9,6 @@ @import 'framework/animations'; @import 'framework/vue_transitions'; -@import 'framework/asciidoctor'; @import 'framework/banner'; @import 'framework/blocks'; @import 'framework/buttons'; diff --git a/app/assets/stylesheets/framework/animations.scss b/app/assets/stylesheets/framework/animations.scss index 6f5a2e561af..6bc5632365f 100644 --- a/app/assets/stylesheets/framework/animations.scss +++ b/app/assets/stylesheets/framework/animations.scss @@ -104,7 +104,7 @@ } .btn { - @include transition(background-color, border-color, color, box-shadow); + @include transition(border-color); } .dropdown-menu-toggle, diff --git a/app/assets/stylesheets/framework/asciidoctor.scss b/app/assets/stylesheets/framework/asciidoctor.scss deleted file mode 100644 index 1586265d40e..00000000000 --- a/app/assets/stylesheets/framework/asciidoctor.scss +++ /dev/null @@ -1,27 +0,0 @@ -.admonitionblock td.icon { - width: 1%; - - [class^='fa icon-'] { - @extend .fa-2x; - } - - .icon-note { - @extend .fa-thumb-tack; - } - - .icon-tip { - @extend .fa-lightbulb-o; - } - - .icon-warning { - @extend .fa-exclamation-triangle; - } - - .icon-caution { - @extend .fa-fire; - } - - .icon-important { - @extend .fa-exclamation-circle; - } -} diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index 767832e242c..e0b6da31261 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -24,12 +24,11 @@ border-radius: $border-radius-default; font-size: $gl-font-size; font-weight: $gl-font-weight-normal; - padding: $gl-vert-padding $gl-btn-padding; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; &:focus, &:active { background-color: $btn-active-gray; - box-shadow: $gl-btn-active-background; } } @@ -50,77 +49,89 @@ color: $text; } - &:hover, - &:focus { - background-color: $hover-background; - border-color: $hover-border; - color: $hover-text; + &:not(:disabled):not(.disabled) { + &:hover { + box-shadow: inset 0 0 0 1px $hover-border, 0 2px 2px 0 $gl-btn-hover-shadow-light; + } - > .icon { - color: $hover-text; + &:focus { + box-shadow: inset 0 0 0 1px $hover-border, 0 0 4px 1px $blue-300; } - } - &:focus { - box-shadow: 0 0 4px 1px $blue-300; - } + &:hover, + &:focus { + background-color: $hover-background; + border-color: $hover-border; + color: $hover-text; - &:active { - background-color: $active-background; - border-color: $active-border; - box-shadow: inset 0 2px 4px 0 rgba($black, 0.2); - color: $active-text; + > .icon { + color: $hover-text; + } + } - > .icon { + &:active, + &:active:focus { + background-color: $active-background; + border-color: $active-border; + box-shadow: inset 0 0 0 1px $hover-border, inset 0 2px 4px 0 rgba($black, 0.2); color: $active-text; - } - &:focus { - box-shadow: inset 0 2px 4px 0 rgba($black, 0.2); + > .icon { + color: $active-text; + } } } } -@mixin btn-color($light, $border-light, $normal, $border-normal, $dark, $border-dark, $color) { +@mixin btn-color($light, $border-light, $normal, $border-normal, $dark, $border-dark, $color, $hover-shadow-color: $gl-btn-hover-shadow-dark) { background-color: $light; border-color: $border-light; color: $color; - &:hover, - &:focus { - background-color: $normal; - border-color: $border-normal; - color: $color; - } + &:not(:disabled):not(.disabled) { + &:hover { + box-shadow: inset 0 0 0 1px $border-normal, 0 2px 2px 0 $hover-shadow-color; + } - &:active, - &.active { - box-shadow: $gl-btn-active-background; + &:focus { + box-shadow: inset 0 0 0 1px $border-normal, 0 0 4px 1px $blue-300; + } - background-color: $dark; - border-color: $border-dark; - color: $color; + &:hover, + &:focus { + background-color: $normal; + border-color: $border-normal; + color: $color; + } + + &:active, + &.active { + box-shadow: inset 0 2px 4px 0 $gl-btn-hover-shadow-dark; + background-color: $dark; + border-color: $border-dark; + color: $color; + } } } @mixin btn-green { - @include btn-color($green-500, $green-600, $green-600, $green-700, $green-700, $green-800, $white-light); + @include btn-color($green-500, $green-600, $green-500, $green-700, $green-600, $green-800, $white-light); } @mixin btn-blue { - @include btn-color($blue-500, $blue-600, $blue-600, $blue-700, $blue-700, $blue-800, $white-light); + @include btn-color($blue-500, $blue-600, $blue-500, $blue-700, $blue-600, $blue-800, $white-light); } @mixin btn-orange { - @include btn-color($orange-500, $orange-600, $orange-600, $orange-700, $orange-700, $orange-800, $white-light); + @include btn-color($orange-500, $orange-600, $orange-500, $orange-700, $orange-600, $orange-800, $white-light); } @mixin btn-red { - @include btn-color($red-500, $red-600, $red-600, $red-700, $red-700, $red-800, $white-light); + @include btn-color($red-500, $red-600, $red-500, $red-700, $red-600, $red-800, $white-light); } @mixin btn-white { - @include btn-color($white-light, $border-color, $white-normal, $border-white-normal, $white-dark, $border-gray-dark, $gl-text-color); + @include btn-color($white-light, $gray-400, $gray-200, $gray-400, $gl-gray-200, $gray-500, $gl-text-color, $gl-btn-hover-shadow-light); } @mixin btn-with-margin { @@ -149,23 +160,22 @@ color: $gl-text-color; white-space: nowrap; + line-height: $gl-btn-line-height; &:focus:active { outline: 0; } - &.btn-sm { - padding: 4px 10px; - font-size: $gl-btn-small-font-size; - line-height: $gl-btn-small-line-height; - } - &.btn-xs { - padding: 2px $gl-btn-padding; font-size: $gl-btn-xs-font-size; line-height: $gl-btn-xs-line-height; } + &.btn-sm, + &.btn-xs { + padding: 3px $gl-bordered-btn-vert-padding; + } + &.btn-success, &.btn-register { @include btn-green; @@ -239,7 +249,7 @@ &.dropdown-toggle { .fa-caret-down { - margin-left: 3px; + margin: 0; } } @@ -272,10 +282,7 @@ } svg { - height: 15px; - width: 15px; - position: relative; - top: 2px; + @include btn-svg; } svg, @@ -330,6 +337,12 @@ &.btn-grouped { @include btn-with-margin; } + + .btn { + border-radius: $border-radius-default; + font-size: $gl-font-size; + line-height: $gl-btn-line-height; + } } .btn-clipboard { @@ -487,18 +500,25 @@ &:active, &:focus { color: $gl-text-color-secondary; + border: 1px solid $border-gray-normal-dashed; background-color: $white-normal; } } -.btn-svg svg { - @include btn-svg; +.btn-svg { + padding: $gl-bordered-btn-vert-padding; + + svg { + @include btn-svg; + display: block; + } } // All disabled buttons, regardless of color, type, etc %disabled { background-color: $gray-light !important; border-color: $gray-200 !important; + box-shadow: none; color: $gl-text-color-disabled !important; opacity: 1 !important; cursor: default !important; diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss index 29f63e9578d..05afcecca05 100644 --- a/app/assets/stylesheets/framework/dropdowns.scss +++ b/app/assets/stylesheets/framework/dropdowns.scss @@ -8,12 +8,6 @@ } } -@mixin chevron-active { - .fa-chevron-down { - color: $gray-darkest; - } -} - @mixin set-visible { transform: translateY(0); display: block; @@ -49,7 +43,6 @@ .dropdown-toggle, .dropdown-menu-toggle { - @include chevron-active; border-color: $gray-darkest; } @@ -65,12 +58,12 @@ .dropdown-toggle, .confidential-merge-request-fork-group .dropdown-toggle { - padding: 6px 8px 6px 10px; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; background-color: $white-light; color: $gl-text-color; font-size: 14px; + line-height: $gl-btn-line-height; text-align: left; - border: 1px solid $border-color; border-radius: $border-radius-base; white-space: nowrap; @@ -103,10 +96,6 @@ padding-right: 25px; } - .fa { - color: $gray-darkest; - } - .fa-chevron-down { font-size: $dropdown-chevron-size; position: relative; @@ -115,12 +104,10 @@ } &:hover { - @include chevron-active; border-color: $gray-darkest; } &:focus:active { - @include chevron-active; border-color: $dropdown-toggle-active-border-color; outline: 0; } diff --git a/app/assets/stylesheets/framework/lists.scss b/app/assets/stylesheets/framework/lists.scss index 954551fef97..460d9ea9526 100644 --- a/app/assets/stylesheets/framework/lists.scss +++ b/app/assets/stylesheets/framework/lists.scss @@ -265,7 +265,6 @@ ul.controls { } .issuable-pipeline-broken a, - .issuable-pipeline-status a, .author-link { display: flex; } diff --git a/app/assets/stylesheets/framework/modal.scss b/app/assets/stylesheets/framework/modal.scss index b721b90fbb3..fd9a75bc5b6 100644 --- a/app/assets/stylesheets/framework/modal.scss +++ b/app/assets/stylesheets/framework/modal.scss @@ -19,16 +19,23 @@ } } - // leave enough space for the close icon .modal-title { + line-height: $gl-line-height-24; + + // leave enough space for the close icon &.mw-100, &.w-100 { - // after upgrading to Bootstrap 4.2 we can use $modal-header-padding-x here - // https://github.com/twbs/bootstrap/pull/26976 - margin-right: -28px; - padding-right: 28px; + margin-right: -$modal-header-padding-x; + padding-right: $modal-header-padding-x; } } + + .close { + font-weight: $gl-font-weight-normal; + line-height: $gl-line-height; + color: $gray-900; + opacity: 1; + } } .modal-body { @@ -63,6 +70,10 @@ margin-left: $grid-size; } + .btn-group .btn + .btn { + margin-left: -1px; + } + @include media-breakpoint-down(xs) { flex-direction: column; @@ -72,6 +83,11 @@ margin-left: 0; margin-top: $grid-size; } + + .btn-group .btn + .btn { + margin-left: -1px; + margin-top: 0; + } } } @@ -93,12 +109,12 @@ body.modal-open { .modal-content { border-radius: $modal-border-radius; - *:first-child { + > :first-child { border-top-left-radius: $modal-border-radius; border-top-right-radius: $modal-border-radius; } - *:last-child { + > :last-child { border-bottom-left-radius: $modal-border-radius; border-bottom-right-radius: $modal-border-radius; } diff --git a/app/assets/stylesheets/framework/typography.scss b/app/assets/stylesheets/framework/typography.scss index 7baab478034..c201605e83d 100644 --- a/app/assets/stylesheets/framework/typography.scss +++ b/app/assets/stylesheets/framework/typography.scss @@ -1,5 +1,5 @@ /** - * Apply Markdown typography + * Apply Markup (Markdown/AsciiDoc) typography * */ .md:not(.use-csslab) { @@ -245,6 +245,21 @@ } } + ul.checklist, + ul.none, + ol.none, + ul.no-bullet, + ol.no-bullet, + ol.unnumbered, + ul.unstyled, + ol.unstyled { + list-style-type: none; + + li { + margin-left: 0; + } + } + li { line-height: 1.6em; margin-left: 25px; @@ -321,6 +336,54 @@ visibility: visible; } } + + .big { + font-size: larger; + } + + .small { + font-size: smaller; + } + + .underline { + text-decoration: underline; + } + + .overline { + text-decoration: overline; + } + + .line-through { + text-decoration: line-through; + } + + .admonitionblock td.icon { + width: 1%; + + [class^='fa icon-'] { + @extend .fa-2x; + } + + .icon-note { + @extend .fa-thumb-tack; + } + + .icon-tip { + @extend .fa-lightbulb-o; + } + + .icon-warning { + @extend .fa-exclamation-triangle; + } + + .icon-caution { + @extend .fa-fire; + } + + .icon-important { + @extend .fa-exclamation-circle; + } + } } diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index c108f45622f..047a9799c3f 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -405,6 +405,8 @@ $tanuki-yellow: #fca326; */ $green-500-focus: rgba($green-500, 0.4); $gl-btn-active-background: rgba(0, 0, 0, 0.16); +$gl-btn-hover-shadow-dark: rgba($black, 0.2); +$gl-btn-hover-shadow-light: rgba($black, 0.1); $gl-btn-active-gradient: inset 0 2px 3px $gl-btn-active-background; /* @@ -481,6 +483,8 @@ $gl-btn-padding: 10px; $gl-btn-line-height: 16px; $gl-btn-vert-padding: 8px; $gl-btn-horz-padding: 12px; +$gl-bordered-btn-vert-padding: $gl-btn-vert-padding - 1px; +$gl-bordered-btn-horz-padding: $gl-btn-horz-padding - 1px; $gl-btn-small-font-size: 13px; $gl-btn-small-line-height: 18px; $gl-btn-xs-font-size: 13px; diff --git a/app/assets/stylesheets/pages/commits.scss b/app/assets/stylesheets/pages/commits.scss index e12ea6fcb99..ffc6e433988 100644 --- a/app/assets/stylesheets/pages/commits.scss +++ b/app/assets/stylesheets/pages/commits.scss @@ -214,10 +214,10 @@ .label, .btn { - padding: $gl-vert-padding $gl-btn-padding; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; border: 1px $border-color solid; font-size: $gl-font-size; - line-height: $line-height-base; + line-height: $gl-btn-line-height; border-radius: 0; display: flex; align-items: center; diff --git a/app/assets/stylesheets/pages/issuable.scss b/app/assets/stylesheets/pages/issuable.scss index 6a0127eb51c..66ea70e79da 100644 --- a/app/assets/stylesheets/pages/issuable.scss +++ b/app/assets/stylesheets/pages/issuable.scss @@ -929,10 +929,6 @@ margin: 0; } } - - .dropdown-toggle > .icon { - margin: 0 3px; - } } .right-sidebar-collapsed { diff --git a/app/assets/stylesheets/pages/issues.scss b/app/assets/stylesheets/pages/issues.scss index 8359a60ec9f..e51ca44476c 100644 --- a/app/assets/stylesheets/pages/issues.scss +++ b/app/assets/stylesheets/pages/issues.scss @@ -267,7 +267,6 @@ ul.related-merge-requests > li { .fa-caret-down { pointer-events: none; color: inherit; - margin-left: 0; } } } diff --git a/app/assets/stylesheets/pages/note_form.scss b/app/assets/stylesheets/pages/note_form.scss index c6bac33e888..1d57a4a4784 100644 --- a/app/assets/stylesheets/pages/note_form.scss +++ b/app/assets/stylesheets/pages/note_form.scss @@ -417,7 +417,6 @@ table { i { color: $white-light; - padding-right: 2px; margin-top: 2px; } diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index c80beceae52..73ba09dbba5 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -429,7 +429,7 @@ padding: 0; background: transparent; border: 0; - line-height: 34px; + line-height: 2; margin: 0; > li + li::before { @@ -792,7 +792,6 @@ .btn { margin-top: $gl-padding; - padding: $gl-btn-vert-padding $gl-btn-padding; line-height: $gl-btn-line-height; .icon { diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index 5664f46484e..5c732ab0d1f 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -90,7 +90,7 @@ .add-to-tree { vertical-align: top; - padding: 8px; + padding: $gl-bordered-btn-vert-padding; svg { top: 0; diff --git a/app/controllers/boards/issues_controller.rb b/app/controllers/boards/issues_controller.rb index 0dd7500623d..353a9806fd1 100644 --- a/app/controllers/boards/issues_controller.rb +++ b/app/controllers/boards/issues_controller.rb @@ -2,16 +2,24 @@ module Boards class IssuesController < Boards::ApplicationController + # This is the maximum amount of issues which can be moved by one request to + # bulk_move for now. This is temporary and might be removed in future by + # introducing an alternative (async?) approach. + # (related: https://gitlab.com/groups/gitlab-org/-/epics/382) + MAX_MOVE_ISSUES_COUNT = 50 + include BoardsResponses include ControllerWithCrossProjectAccessCheck requires_cross_project_access if: -> { board&.group_board? } - before_action :whitelist_query_limiting, only: [:index, :update] + before_action :whitelist_query_limiting, only: [:index, :update, :bulk_move] before_action :authorize_read_issue, only: [:index] before_action :authorize_create_issue, only: [:create] before_action :authorize_update_issue, only: [:update] skip_before_action :authenticate_user!, only: [:index] + before_action :validate_id_list, only: [:bulk_move] + before_action :can_move_issues?, only: [:bulk_move] # rubocop: disable CodeReuse/ActiveRecord def index @@ -46,6 +54,17 @@ module Boards end end + def bulk_move + service = Boards::Issues::MoveService.new(board_parent, current_user, move_params(true)) + + issues = Issue.find(params[:ids]) + if service.execute_multiple(issues) + head :ok + else + head :unprocessable_entity + end + end + def update service = Boards::Issues::MoveService.new(board_parent, current_user, move_params) @@ -58,6 +77,10 @@ module Boards private + def can_move_issues? + head(:forbidden) unless can?(current_user, :admin_issue, board) + end + def render_issues(issues, metadata) data = { issues: serialize_as_json(issues) } data.merge!(metadata) @@ -90,8 +113,9 @@ module Boards end end - def move_params - params.permit(:board_id, :id, :from_list_id, :to_list_id, :move_before_id, :move_after_id) + def move_params(multiple = false) + id_param = multiple ? :ids : :id + params.permit(id_param, :board_id, :from_list_id, :to_list_id, :move_before_id, :move_after_id) end def issue_params @@ -112,5 +136,10 @@ module Boards # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42439 Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42428') end + + def validate_id_list + head(:bad_request) unless params[:ids].is_a?(Array) + head(:unprocessable_entity) if params[:ids].size > MAX_MOVE_ISSUES_COUNT + end end end diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 8d518c14b90..2985da35d83 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,35 +1,36 @@ # frozen_string_literal: true class ChaosController < ActionController::Base - before_action :validate_request + before_action :validate_chaos_secret, unless: :development? + before_action :request_start_time def leakmem - memory_mb = (params[:memory_mb]&.to_i || 100) - duration_s = (params[:duration_s]&.to_i || 30).seconds - - start = Time.now retainer = [] # Add `n` 1mb chunks of memory to the retainer array memory_mb.times { retainer << "x" * 1.megabyte } - duration_taken = (Time.now - start).seconds - Kernel.sleep duration_s - duration_taken if duration_s > duration_taken + Kernel.sleep(duration_left) render plain: "OK" end - def cpuspin - duration_s = (params[:duration_s]&.to_i || 30).seconds - end_time = Time.now + duration_s.seconds - - rand while Time.now < end_time + def cpu_spin + rand while Time.now < expected_end_time render plain: "OK" end + def db_spin + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + def sleep - duration_s = (params[:duration_s]&.to_i || 30).seconds - Kernel.sleep duration_s + Kernel.sleep(duration_left) render plain: "OK" end @@ -40,17 +41,57 @@ class ChaosController < ActionController::Base private - def validate_request - secret = ENV['GITLAB_CHAOS_SECRET'] - # GITLAB_CHAOS_SECRET is required unless you're running in Development mode - if !secret && !Rails.env.development? - render plain: "chaos misconfigured: please configure GITLAB_CHAOS_SECRET when using GITLAB_ENABLE_CHAOS_ENDPOINTS outside of a development environment", status: :internal_server_error - end + def request_start_time + @start_time ||= Time.now + end + + def expected_end_time + request_start_time + duration_s + end - return unless secret + def duration_left + # returns 0 if over time + [expected_end_time - Time.now, 0].max + end + + def validate_chaos_secret + unless chaos_secret_configured + render plain: "chaos misconfigured: please configure GITLAB_CHAOS_SECRET", + status: :internal_server_error + return + end - unless request.headers["HTTP_X_CHAOS_SECRET"] == secret - render plain: "To experience chaos, please set X-Chaos-Secret header", status: :unauthorized + unless Devise.secure_compare(chaos_secret_configured, chaos_secret_request) + render plain: "To experience chaos, please set a valid `X-Chaos-Secret` header or `token` param", + status: :unauthorized + return end end + + def chaos_secret_configured + ENV['GITLAB_CHAOS_SECRET'] + end + + def chaos_secret_request + request.headers["HTTP_X_CHAOS_SECRET"] || params[:token] + end + + def interval_s + interval_s = params[:interval_s] || 1 + interval_s.to_f.seconds + end + + def duration_s + duration_s = params[:duration_s] || 30 + duration_s.to_i.seconds + end + + def memory_mb + memory_mb = params[:memory_mb] || 100 + memory_mb.to_i + end + + def development? + Rails.env.development? + end end diff --git a/app/controllers/concerns/requires_whitelisted_monitoring_client.rb b/app/controllers/concerns/requires_whitelisted_monitoring_client.rb index f47ead2f0da..2e9905997db 100644 --- a/app/controllers/concerns/requires_whitelisted_monitoring_client.rb +++ b/app/controllers/concerns/requires_whitelisted_monitoring_client.rb @@ -28,7 +28,7 @@ module RequiresWhitelistedMonitoringClient def valid_token? token = params[:token].presence || request.headers['TOKEN'] token.present? && - ActiveSupport::SecurityUtils.variable_size_secure_compare( + ActiveSupport::SecurityUtils.secure_compare( token, Gitlab::CurrentSettings.health_check_access_token ) diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 797833e3f91..dbddee47997 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -107,7 +107,7 @@ class GroupsController < Groups::ApplicationController if Groups::UpdateService.new(@group, current_user, group_params).execute redirect_to edit_group_path(@group, anchor: params[:update_section]), notice: "Group '#{@group.name}' was successfully updated." else - @group.restore_path! + @group.path = @group.path_before_last_save || @group.path_was render action: "edit" end diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index ecf05e6ea64..1bca52106fa 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -13,6 +13,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController before_action only: [:metrics, :additional_metrics, :metrics_dashboard] do push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint) push_frontend_feature_flag(:environment_metrics_show_multiple_dashboards) + push_frontend_feature_flag(:environment_metrics_additional_panel_types) push_frontend_feature_flag(:prometheus_computed_alerts) end diff --git a/app/graphql/mutations/award_emojis/base.rb b/app/graphql/mutations/award_emojis/base.rb index d868db84f9d..583744c3884 100644 --- a/app/graphql/mutations/award_emojis/base.rb +++ b/app/graphql/mutations/award_emojis/base.rb @@ -3,8 +3,6 @@ module Mutations module AwardEmojis class Base < BaseMutation - include Gitlab::Graphql::Authorize::AuthorizeResource - authorize :award_emoji argument :awardable_id, diff --git a/app/graphql/mutations/base_mutation.rb b/app/graphql/mutations/base_mutation.rb index 08d2a1f18a3..7273a74cb86 100644 --- a/app/graphql/mutations/base_mutation.rb +++ b/app/graphql/mutations/base_mutation.rb @@ -2,6 +2,7 @@ module Mutations class BaseMutation < GraphQL::Schema::RelayClassicMutation + prepend Gitlab::Graphql::Authorize::AuthorizeResource prepend Gitlab::Graphql::CopyFieldDescription field :errors, [GraphQL::STRING_TYPE], diff --git a/app/graphql/mutations/merge_requests/base.rb b/app/graphql/mutations/merge_requests/base.rb index e85d16fc2c5..28e0cdc8cc7 100644 --- a/app/graphql/mutations/merge_requests/base.rb +++ b/app/graphql/mutations/merge_requests/base.rb @@ -3,7 +3,6 @@ module Mutations module MergeRequests class Base < BaseMutation - include Gitlab::Graphql::Authorize::AuthorizeResource include Mutations::ResolvesProject argument :project_path, GraphQL::ID_TYPE, diff --git a/app/graphql/mutations/notes/base.rb b/app/graphql/mutations/notes/base.rb index a7198f5fba6..31dabc0a660 100644 --- a/app/graphql/mutations/notes/base.rb +++ b/app/graphql/mutations/notes/base.rb @@ -3,8 +3,6 @@ module Mutations module Notes class Base < BaseMutation - include Gitlab::Graphql::Authorize::AuthorizeResource - field :note, Types::Notes::NoteType, null: true, diff --git a/app/helpers/boards_helper.rb b/app/helpers/boards_helper.rb index 8ef68018d23..bbe05f40999 100644 --- a/app/helpers/boards_helper.rb +++ b/app/helpers/boards_helper.rb @@ -99,7 +99,11 @@ module BoardsHelper recent_project_boards_path(@project) if current_board_parent.is_a?(Project) end + def serializer + CurrentBoardSerializer.new + end + def current_board_json - board.to_json + serializer.represent(board).as_json end end diff --git a/app/helpers/dropdowns_helper.rb b/app/helpers/dropdowns_helper.rb index 64c5fae7d96..fd94f07cc2c 100644 --- a/app/helpers/dropdowns_helper.rb +++ b/app/helpers/dropdowns_helper.rb @@ -46,7 +46,7 @@ module DropdownsHelper def dropdown_toggle(toggle_text, data_attr, options = {}) default_label = data_attr[:default_label] - content_tag(:button, disabled: options[:disabled], class: "dropdown-menu-toggle #{options[:toggle_class] if options.key?(:toggle_class)}", id: (options[:id] if options.key?(:id)), type: "button", data: data_attr) do + content_tag(:button, disabled: options[:disabled], class: "dropdown-menu-toggle btn #{options[:toggle_class] if options.key?(:toggle_class)}", id: (options[:id] if options.key?(:id)), type: "button", data: data_attr) do output = content_tag(:span, toggle_text, class: "dropdown-toggle-text #{'is-default' if toggle_text == default_label}") output << icon('chevron-down') output.html_safe diff --git a/app/helpers/markup_helper.rb b/app/helpers/markup_helper.rb index 8ccb39f8444..d76a0f3a3b8 100644 --- a/app/helpers/markup_helper.rb +++ b/app/helpers/markup_helper.rb @@ -3,7 +3,7 @@ require 'nokogiri' module MarkupHelper - include ActionView::Helpers::TagHelper + include ActionView::Helpers::TextHelper include ::Gitlab::ActionViewOutput::Context def plain?(filename) @@ -154,9 +154,7 @@ module MarkupHelper elsif asciidoc?(file_name) asciidoc_unsafe(text, context) elsif plain?(file_name) - content_tag :pre, class: 'plain-readme' do - text - end + plain_unsafe(text) else other_markup_unsafe(file_name, text, context) end @@ -271,6 +269,12 @@ module MarkupHelper Gitlab::Asciidoc.render(text, context) end + def plain_unsafe(text) + content_tag :pre, class: 'plain-readme' do + text + end + end + def other_markup_unsafe(file_name, text, context = {}) Gitlab::OtherMarkup.render(file_name, text, context) end diff --git a/app/helpers/sorting_helper.rb b/app/helpers/sorting_helper.rb index 26692934456..d5e5b472115 100644 --- a/app/helpers/sorting_helper.rb +++ b/app/helpers/sorting_helper.rb @@ -31,22 +31,24 @@ module SortingHelper end def projects_sort_options_hash - Feature.enabled?(:project_list_filter_bar) && !current_controller?('admin/projects') ? projects_sort_common_options_hash : old_projects_sort_options_hash - end + use_old_sorting = Feature.disabled?(:project_list_filter_bar) || current_controller?('admin/projects') - # TODO: Simplify these sorting options - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60798 - # https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/11209#note_162234858 - def old_projects_sort_options_hash options = { sort_value_latest_activity => sort_title_latest_activity, + sort_value_recently_created => sort_title_created_date, sort_value_name => sort_title_name, - sort_value_oldest_activity => sort_title_oldest_activity, - sort_value_oldest_created => sort_title_oldest_created, - sort_value_recently_created => sort_title_recently_created, - sort_value_stars_desc => sort_title_most_stars + sort_value_stars_desc => sort_title_stars } + if use_old_sorting + options = options.merge({ + sort_value_oldest_activity => sort_title_oldest_activity, + sort_value_oldest_created => sort_title_oldest_created, + sort_value_recently_created => sort_title_recently_created, + sort_value_stars_desc => sort_title_most_stars + }) + end + if current_controller?('admin/projects') options[sort_value_largest_repo] = sort_title_largest_repo end @@ -54,26 +56,14 @@ module SortingHelper options end - def projects_sort_common_options_hash - { - sort_value_latest_activity => sort_title_latest_activity, - sort_value_recently_created => sort_title_created_date, - sort_value_name => sort_title_name, - sort_value_stars_desc => sort_title_stars - } - end - def projects_sort_option_titles - { - sort_value_latest_activity => sort_title_latest_activity, - sort_value_recently_created => sort_title_created_date, - sort_value_name => sort_title_name, - sort_value_stars_desc => sort_title_stars, + # Only used for the project filter search bar + projects_sort_options_hash.merge({ sort_value_oldest_activity => sort_title_latest_activity, sort_value_oldest_created => sort_title_created_date, sort_value_name_desc => sort_title_name, sort_value_stars_asc => sort_title_stars - } + }) end def projects_reverse_sort_options_hash @@ -210,47 +200,42 @@ module SortingHelper sort_options_hash[sort_value] end - def issuable_sort_icon_suffix(sort_value) + def sort_direction_icon(sort_value) case sort_value when sort_value_milestone, sort_value_due_date, /_asc\z/ - 'lowest' + 'sort-lowest' else - 'highest' + 'sort-highest' end end - # TODO: dedupicate issuable and project sort direction - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60798 - def issuable_sort_direction_button(sort_value) + def sort_direction_button(reverse_url, reverse_sort, sort_value) link_class = 'btn btn-default has-tooltip reverse-sort-btn qa-reverse-sort' - reverse_sort = issuable_reverse_sort_order_hash[sort_value] + icon = sort_direction_icon(sort_value) + url = reverse_url - if reverse_sort - reverse_url = page_filter_path(sort: reverse_sort) - else - reverse_url = '#' + unless reverse_sort + url = '#' link_class += ' disabled' end - link_to(reverse_url, type: 'button', class: link_class, title: s_('SortOptions|Sort direction')) do - sprite_icon("sort-#{issuable_sort_icon_suffix(sort_value)}", size: 16) + link_to(url, type: 'button', class: link_class, title: s_('SortOptions|Sort direction')) do + sprite_icon(icon, size: 16) end end + def issuable_sort_direction_button(sort_value) + reverse_sort = issuable_reverse_sort_order_hash[sort_value] + url = page_filter_path(sort: reverse_sort) + + sort_direction_button(url, reverse_sort, sort_value) + end + def project_sort_direction_button(sort_value) - link_class = 'btn btn-default has-tooltip reverse-sort-btn qa-reverse-sort' reverse_sort = projects_reverse_sort_options_hash[sort_value] + url = filter_projects_path(sort: reverse_sort) - if reverse_sort - reverse_url = filter_projects_path(sort: reverse_sort) - else - reverse_url = '#' - link_class += ' disabled' - end - - link_to(reverse_url, type: 'button', class: link_class, title: s_('SortOptions|Sort direction')) do - sprite_icon("sort-#{issuable_sort_icon_suffix(sort_value)}", size: 16) - end + sort_direction_button(url, reverse_sort, sort_value) end # Titles. diff --git a/app/helpers/tree_helper.rb b/app/helpers/tree_helper.rb index 4690b6ffbe1..a3575462de0 100644 --- a/app/helpers/tree_helper.rb +++ b/app/helpers/tree_helper.rb @@ -147,4 +147,11 @@ module TreeHelper def relative_url_root Gitlab.config.gitlab.relative_url_root.presence || '/' end + + # project and path are used on the EE version + def tree_content_data(logs_path, project, path) + { + "logs-path" => logs_path + } + end end diff --git a/app/models/chat_team.rb b/app/models/chat_team.rb index 52b5a7b4a91..28aab279545 100644 --- a/app/models/chat_team.rb +++ b/app/models/chat_team.rb @@ -12,6 +12,6 @@ class ChatTeam < ApplicationRecord # Either the group is not found, or the user doesn't have the proper # access on the mattermost instance. In the first case, we're done either way # in the latter case, we can't recover by retrying, so we just log what happened - Rails.logger.error("Mattermost team deletion failed: #{e}") + Rails.logger.error("Mattermost team deletion failed: #{e}") # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 89cc082d0bc..635fcc86166 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -266,7 +266,7 @@ module Ci begin Ci::Build.retry(build, build.user) rescue Gitlab::Access::AccessDeniedError => ex - Rails.logger.error "Unable to auto-retry job #{build.id}: #{ex}" + Rails.logger.error "Unable to auto-retry job #{build.id}: #{ex}" # rubocop:disable Gitlab/RailsLogger end end end @@ -578,7 +578,7 @@ module Ci end def valid_token?(token) - self.token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) + self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end def has_tags? diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 20ca4a9ab24..2262282e647 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -196,7 +196,7 @@ module Ci sql = 'CASE ci_pipelines.source WHEN (?) THEN 0 ELSE 1 END, ci_pipelines.id DESC' query = ApplicationRecord.send(:sanitize_sql_array, [sql, sources[:merge_request_event]]) # rubocop:disable GitlabSecurity/PublicSend - order(query) + order(Arel.sql(query)) end scope :for_user, -> (user) { where(user: user) } diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 42203a5f214..9713e79f525 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -87,6 +87,16 @@ module CacheMarkdownField __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end + # Updates the markdown cache if necessary, then returns the field + # Unlike `cached_html_for` it returns `nil` if the field does not exist + def updated_cached_html_for(markdown_field) + return unless cached_markdown_fields.markdown_fields.include?(markdown_field) + + refresh_markdown_cache if attribute_invalidated?(cached_markdown_fields.html_field(markdown_field)) + + cached_html_for(markdown_field) + end + def latest_cached_markdown_version @latest_cached_markdown_version ||= (Gitlab::MarkdownCache::CACHE_COMMONMARK_VERSION << 16) | local_version end @@ -139,8 +149,9 @@ module CacheMarkdownField # The HTML becomes invalid if any dependent fields change. For now, assume # author and project invalidate the cache in all circumstances. define_method(invalidation_method) do - invalidations = changed_markdown_fields & [markdown_field.to_s, *INVALIDATED_BY] - invalidations.delete(markdown_field.to_s) if changed_markdown_fields.include?("#{markdown_field}_html") + changed_fields = changed_attributes.keys + invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] + invalidations.delete(markdown_field.to_s) if changed_fields.include?("#{markdown_field}_html") !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end end diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb index 8cbf4bcfaf7..53dff2adfc3 100644 --- a/app/models/concerns/cacheable_attributes.rb +++ b/app/models/concerns/cacheable_attributes.rb @@ -49,7 +49,7 @@ module CacheableAttributes current_without_cache.tap { |current_record| current_record&.cache! } rescue => e if Rails.env.production? - Rails.logger.warn("Cached record for #{name} couldn't be loaded, falling back to uncached record: #{e}") + Rails.logger.warn("Cached record for #{name} couldn't be loaded, falling back to uncached record: #{e}") # rubocop:disable Gitlab/RailsLogger else raise e end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index e1d5ce7f7d4..91dda803031 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -59,6 +59,7 @@ module Ci variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) + variables.append(key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s) diff --git a/app/models/concerns/from_union.rb b/app/models/concerns/from_union.rb index 9b8595b1211..e28dee34815 100644 --- a/app/models/concerns/from_union.rb +++ b/app/models/concerns/from_union.rb @@ -40,11 +40,7 @@ module FromUnion .new(members, remove_duplicates: remove_duplicates) .to_sql - # This pattern is necessary as a bug in Rails 4 can cause the use of - # `from("string here").includes(:foo)` to break ActiveRecord. This is - # fixed in https://github.com/rails/rails/pull/25374, which is released as - # part of Rails 5. - from([Arel.sql("(#{union}) #{alias_as}")]) + from(Arel.sql("(#{union}) #{alias_as}")) end # rubocop: enable Gitlab/Union end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 78bcce2f592..27a5c3d5286 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -33,22 +33,24 @@ module HasStatus canceled = scope_relevant.canceled.select('count(*)').to_sql warnings = scope_warnings.select('count(*) > 0').to_sql.presence || 'false' - "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success}) THEN 'success' - WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{preparing}) THEN 'preparing' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' - WHEN (#{running})+(#{pending})>0 THEN 'running' - WHEN (#{manual})>0 THEN 'manual' - WHEN (#{scheduled})>0 THEN 'scheduled' - WHEN (#{preparing})>0 THEN 'preparing' - WHEN (#{created})>0 THEN 'running' - ELSE 'failed' - END)" + Arel.sql( + "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' + WHEN (#{builds})=(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{preparing}) THEN 'preparing' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})>0 THEN 'running' + WHEN (#{manual})>0 THEN 'manual' + WHEN (#{scheduled})>0 THEN 'scheduled' + WHEN (#{preparing})>0 THEN 'preparing' + WHEN (#{created})>0 THEN 'running' + ELSE 'failed' + END)" + ) end def status @@ -88,22 +90,22 @@ module HasStatus state :scheduled, value: 'scheduled' end - scope :created, -> { where(status: 'created') } - scope :preparing, -> { where(status: 'preparing') } - scope :relevant, -> { where(status: AVAILABLE_STATUSES - ['created']) } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - scope :scheduled, -> { where(status: 'scheduled') } - scope :alive, -> { where(status: [:created, :preparing, :pending, :running]) } - scope :created_or_pending, -> { where(status: [:created, :pending]) } - scope :running_or_pending, -> { where(status: [:running, :pending]) } - scope :finished, -> { where(status: [:success, :failed, :canceled]) } - scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } + scope :created, -> { with_status(:created) } + scope :preparing, -> { with_status(:preparing) } + scope :relevant, -> { without_status(:created) } + scope :running, -> { with_status(:running) } + scope :pending, -> { with_status(:pending) } + scope :success, -> { with_status(:success) } + scope :failed, -> { with_status(:failed) } + scope :canceled, -> { with_status(:canceled) } + scope :skipped, -> { with_status(:skipped) } + scope :manual, -> { with_status(:manual) } + scope :scheduled, -> { with_status(:scheduled) } + scope :alive, -> { with_status(:created, :preparing, :pending, :running) } + scope :created_or_pending, -> { with_status(:created, :pending) } + scope :running_or_pending, -> { with_status(:running, :pending) } + scope :finished, -> { with_status(:success, :failed, :canceled) } + scope :failed_or_canceled, -> { with_status(:failed, :canceled) } scope :cancelable, -> do where(status: [:running, :preparing, :pending, :created, :scheduled]) diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 0d88b34fb48..2f3f9b399d9 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -63,6 +63,9 @@ module Mentionable skip_project_check: skip_project_check? ).merge(mentionable_params) + cached_html = self.try(:updated_cached_html_for, attr.to_sym) + options[:rendered] = cached_html if cached_html + extractor.analyze(text, options) end diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index 22b6b1d720c..e4fe46d722a 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -179,7 +179,7 @@ module RelativePositioning relation = yield relation if block_given? relation - .pluck(self.class.parent_column, "#{calculation}(relative_position) AS position") + .pluck(self.class.parent_column, Arel.sql("#{calculation}(relative_position) AS position")) .first&. last end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index b9ffc64e4a9..9becab632f3 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -46,7 +46,7 @@ module Routable # See https://gitlab.com/gitlab-org/gitlab-ce/issues/18603. Also note that # our unique index is case-sensitive in Postgres. binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = Arel.sql("(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") found = where_full_path_in([path]).reorder(order_sql).take return found if found diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index a15dc19e07a..78544405c49 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -64,7 +64,7 @@ module Storage unless gitlab_shell.mv_namespace(repository_storage, full_path_before_last_save, full_path) - Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_before_last_save} to #{full_path}" + Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_before_last_save} to #{full_path}" # rubocop:disable Gitlab/RailsLogger # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 8c769be0489..1293df571a3 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -52,7 +52,7 @@ module TokenAuthenticatable mod.define_method("#{token_field}_matches?") do |other_token| token = read_attribute(token_field) - token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare(other_token, token) + token.present? && ActiveSupport::SecurityUtils.secure_compare(other_token, token) end end diff --git a/app/models/email.rb b/app/models/email.rb index 0ddaa049c3b..580633d3232 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -4,9 +4,8 @@ class Email < ApplicationRecord include Sortable include Gitlab::SQL::Pattern - belongs_to :user + belongs_to :user, optional: false - validates :user_id, presence: true validates :email, presence: true, uniqueness: true, devise_email: true validate :unique_email, if: ->(email) { email.email_changed? } diff --git a/app/models/environment_status.rb b/app/models/environment_status.rb index 465a42759df..d7dc64190d6 100644 --- a/app/models/environment_status.rb +++ b/app/models/environment_status.rb @@ -40,7 +40,7 @@ class EnvironmentStatus end def changes - return [] if project.route_map_for(sha).nil? + return [] unless has_route_map? changed_files.map { |file| build_change(file) }.compact end @@ -50,6 +50,10 @@ class EnvironmentStatus .merge_request_diff_files.where(deleted_file: false) end + def has_route_map? + project.route_map_for(sha).present? + end + private PAGE_EXTENSIONS = /\A\.(s?html?|php|asp|cgi|pl)\z/i.freeze diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb index 22cedf57b86..5c53cfd8c27 100644 --- a/app/models/merge_requests_closing_issues.rb +++ b/app/models/merge_requests_closing_issues.rb @@ -25,7 +25,7 @@ class MergeRequestsClosingIssues < ApplicationRecord class << self def count_for_collection(ids, current_user) - closing_merge_requests(ids, current_user).group(:issue_id).pluck('issue_id', 'COUNT(*) as count') + closing_merge_requests(ids, current_user).group(:issue_id).pluck('issue_id', Arel.sql('COUNT(*) as count')) end def count_for_issue(id, current_user) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index af50293a179..1d95590bac9 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -41,8 +41,7 @@ class Namespace < ApplicationRecord validates :owner, presence: true, unless: ->(n) { n.type == "Group" } validates :name, presence: true, - length: { maximum: 255 }, - namespace_name: true + length: { maximum: 255 } validates :description, length: { maximum: 255 } validates :path, diff --git a/app/models/project.rb b/app/models/project.rb index bfc35b77b8f..2a29cb1ae09 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -357,7 +357,7 @@ class Project < ApplicationRecord scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) } # last_activity_at is throttled every minute, but last_repository_updated_at is updated with every push - scope :sorted_by_activity, -> { reorder("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC") } + scope :sorted_by_activity, -> { reorder(Arel.sql("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC")) } scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) } scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) } @@ -612,7 +612,7 @@ class Project < ApplicationRecord end end - def initialize(attributes = {}) + def initialize(attributes = nil) # We can't use default_value_for because the database has a default # value of 0 for visibility_level. If someone attempts to create a # private project, default_value_for will assume that the @@ -622,6 +622,8 @@ class Project < ApplicationRecord # # To fix the problem, we assign the actual default in the application if # no explicit visibility has been initialized. + attributes ||= {} + unless visibility_attribute_present?(attributes) attributes[:visibility_level] = Gitlab::CurrentSettings.default_project_visibility end @@ -688,10 +690,6 @@ class Project < ApplicationRecord { scope: :project, status: auto_devops&.enabled || Feature.enabled?(:force_autodevops_on_by_default, self) } end - def multiple_mr_assignees_enabled? - Feature.enabled?(:multiple_merge_request_assignees, self) - end - def daily_statistics_enabled? Feature.enabled?(:project_daily_statistics, self, default_enabled: true) end @@ -784,6 +782,7 @@ class Project < ApplicationRecord job_id end + # rubocop:disable Gitlab/RailsLogger def log_import_activity(job_id, type: :import) job_type = type.to_s.capitalize @@ -793,6 +792,7 @@ class Project < ApplicationRecord Rails.logger.error("#{job_type} job failed to create for #{full_path}.") end end + # rubocop:enable Gitlab/RailsLogger def reset_cache_and_import_attrs run_after_commit do @@ -1555,7 +1555,7 @@ class Project < ApplicationRecord end def valid_runners_token?(token) - self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) + self.runners_token && ActiveSupport::SecurityUtils.secure_compare(token, self.runners_token) end # rubocop: disable CodeReuse/ServiceClass @@ -1665,6 +1665,7 @@ class Project < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass + # rubocop:disable Gitlab/RailsLogger def write_repository_config(gl_full_path: full_path) # We'd need to keep track of project full path otherwise directory tree # created with hashed storage enabled cannot be usefully imported using @@ -1674,6 +1675,7 @@ class Project < ApplicationRecord Rails.logger.error("Error writing to .git/config for project #{full_path} (#{id}): #{e.message}.") nil end + # rubocop:enable Gitlab/RailsLogger def after_import repository.after_import @@ -1715,6 +1717,7 @@ class Project < ApplicationRecord @pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self) end + # rubocop:disable Gitlab/RailsLogger def add_export_job(current_user:, after_export_strategy: nil, params: {}) job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) @@ -1724,6 +1727,7 @@ class Project < ApplicationRecord Rails.logger.error "Export job failed to start for project ID #{self.id}" end end + # rubocop:enable Gitlab/RailsLogger def import_export_shared @import_export_shared ||= Gitlab::ImportExport::Shared.new(self) @@ -1914,9 +1918,8 @@ class Project < ApplicationRecord @route_maps_by_commit ||= Hash.new do |h, sha| h[sha] = begin data = repository.route_map_for(sha) - next unless data - Gitlab::RouteMap.new(data) + Gitlab::RouteMap.new(data) if data rescue Gitlab::RouteMap::FormatError nil end diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index 1605345efd5..23adffb33d8 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -65,7 +65,7 @@ class ProjectImportState < ApplicationRecord update_column(:last_error, sanitized_message) rescue ActiveRecord::ActiveRecordError => e - Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}") + Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}") # rubocop:disable Gitlab/RailsLogger ensure @errors = original_errors end diff --git a/app/models/project_services/ci_service.rb b/app/models/project_services/ci_service.rb index f0ef2d925ab..47106d7bdbb 100644 --- a/app/models/project_services/ci_service.rb +++ b/app/models/project_services/ci_service.rb @@ -7,7 +7,7 @@ class CiService < Service default_value_for :category, 'ci' def valid_token?(token) - self.respond_to?(:token) && self.token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) + self.respond_to?(:token) && self.token.present? && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end def self.supported_events diff --git a/app/models/project_services/slash_commands_service.rb b/app/models/project_services/slash_commands_service.rb index bfabc6d262c..5f5cff97808 100644 --- a/app/models/project_services/slash_commands_service.rb +++ b/app/models/project_services/slash_commands_service.rb @@ -12,7 +12,7 @@ class SlashCommandsService < Service def valid_token?(token) self.respond_to?(:token) && self.token.present? && - ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.token) + ActiveSupport::SecurityUtils.secure_compare(token, self.token) end def self.supported_events diff --git a/app/models/repository.rb b/app/models/repository.rb index a25d5abfa64..187382ad182 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -273,7 +273,7 @@ class Repository # This will still fail if the file is corrupted (e.g. 0 bytes) raw_repository.write_ref(keep_around_ref_name(sha), sha) rescue Gitlab::Git::CommandError => ex - Rails.logger.error "Unable to create keep-around reference for repository #{disk_path}: #{ex}" + Rails.logger.error "Unable to create keep-around reference for repository #{disk_path}: #{ex}" # rubocop:disable Gitlab/RailsLogger end end @@ -934,6 +934,7 @@ class Repository async_remove_remote(remote_name) if tmp_remote_name end + # rubocop:disable Gitlab/RailsLogger def async_remove_remote(remote_name) return unless remote_name @@ -947,6 +948,7 @@ class Repository job_id end + # rubocop:enable Gitlab/RailsLogger def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) diff --git a/app/models/ssh_host_key.rb b/app/models/ssh_host_key.rb index b6fb39ee81f..9bd35d30845 100644 --- a/app/models/ssh_host_key.rb +++ b/app/models/ssh_host_key.rb @@ -106,7 +106,7 @@ class SshHostKey if status.success? && !errors.present? { known_hosts: known_hosts } else - Rails.logger.debug("Failed to detect SSH host keys for #{id}: #{errors}") + Rails.logger.debug("Failed to detect SSH host keys for #{id}: #{errors}") # rubocop:disable Gitlab/RailsLogger { error: 'Failed to detect SSH host keys' } end diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index b483c677be9..928c773c307 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -41,7 +41,7 @@ module Storage gitlab_shell.mv_repository(repository_storage, "#{old_full_path}.wiki", "#{new_full_path}.wiki") return true rescue => e - Rails.logger.error "Exception renaming #{old_full_path} -> #{new_full_path}: #{e}" + Rails.logger.error "Exception renaming #{old_full_path} -> #{new_full_path}: #{e}" # rubocop:disable Gitlab/RailsLogger # Returning false does not rollback after_* transaction but gives # us information about failing some of tasks return false diff --git a/app/models/uploads/base.rb b/app/models/uploads/base.rb index f9814159958..29f376670da 100644 --- a/app/models/uploads/base.rb +++ b/app/models/uploads/base.rb @@ -7,7 +7,7 @@ module Uploads attr_reader :logger def initialize(logger: nil) - @logger ||= Rails.logger + @logger ||= Rails.logger # rubocop:disable Gitlab/RailsLogger end def delete_keys_async(keys_to_delete) diff --git a/app/models/user.rb b/app/models/user.rb index 26be197209a..02637b70f03 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1117,9 +1117,10 @@ class User < ApplicationRecord def ensure_namespace_correct if namespace - namespace.path = namespace.name = username if username_changed? + namespace.path = username if username_changed? + namespace.name = name if name_changed? else - build_namespace(path: username, name: username) + build_namespace(path: username, name: name) end end diff --git a/app/policies/clusters/instance_policy.rb b/app/policies/clusters/instance_policy.rb index f72096e8fc6..bd7ff413afe 100644 --- a/app/policies/clusters/instance_policy.rb +++ b/app/policies/clusters/instance_policy.rb @@ -2,11 +2,6 @@ module Clusters class InstancePolicy < BasePolicy - include ClusterableActions - - condition(:has_clusters, scope: :subject) { clusterable_has_clusters? } - condition(:can_have_multiple_clusters) { multiple_clusters_available? } - rule { admin }.policy do enable :read_cluster enable :add_cluster @@ -14,7 +9,5 @@ module Clusters enable :update_cluster enable :admin_cluster end - - rule { ~can_have_multiple_clusters & has_clusters }.prevent :add_cluster end end diff --git a/app/policies/concerns/clusterable_actions.rb b/app/policies/concerns/clusterable_actions.rb deleted file mode 100644 index 08ddd742ea9..00000000000 --- a/app/policies/concerns/clusterable_actions.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module ClusterableActions - private - - # Overridden on EE module - def multiple_clusters_available? - false - end - - def clusterable_has_clusters? - !subject.clusters.empty? - end -end diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index ea86858181d..9219283992f 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class GroupPolicy < BasePolicy - include ClusterableActions - desc "Group is public" with_options scope: :subject, score: 0 condition(:public_group) { @subject.public? } @@ -29,9 +27,6 @@ class GroupPolicy < BasePolicy GroupProjectsFinder.new(group: @subject, current_user: @user, options: { include_subgroups: true, only_owned: true }).execute.any? end - condition(:has_clusters, scope: :subject) { clusterable_has_clusters? } - condition(:can_have_multiple_clusters) { multiple_clusters_available? } - with_options scope: :subject, score: 0 condition(:request_access_enabled) { @subject.request_access_enabled } @@ -121,8 +116,6 @@ class GroupPolicy < BasePolicy rule { owner & (~share_with_group_locked | ~has_parent | ~parent_share_with_group_locked | can_change_parent_share_with_group_lock) }.enable :change_share_with_group_lock - rule { ~can_have_multiple_clusters & has_clusters }.prevent :add_cluster - rule { developer & developer_maintainer_access }.enable :create_projects rule { create_projects_disabled }.prevent :create_projects diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3c9ffbb2065..e79bac6bee3 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -2,7 +2,6 @@ class ProjectPolicy < BasePolicy extend ClassMethods - include ClusterableActions READONLY_FEATURES_WHEN_ARCHIVED = %i[ issue @@ -114,9 +113,6 @@ class ProjectPolicy < BasePolicy @subject.feature_available?(:merge_requests, @user) end - condition(:has_clusters, scope: :subject) { clusterable_has_clusters? } - condition(:can_have_multiple_clusters) { multiple_clusters_available? } - condition(:internal_builds_disabled) do !@subject.builds_enabled? end @@ -430,8 +426,6 @@ class ProjectPolicy < BasePolicy (~guest & can?(:read_project_for_iids) & merge_requests_visible_to_user) | can?(:read_merge_request) end.enable :read_merge_request_iid - rule { ~can_have_multiple_clusters & has_clusters }.prevent :add_cluster - rule { ~can?(:read_cross_project) & ~classification_label_authorized }.policy do # Preventing access here still allows the projects to be listed. Listing # projects doesn't check the `:read_project` ability. But instead counts diff --git a/app/presenters/clusterable_presenter.rb b/app/presenters/clusterable_presenter.rb index 34bdf156623..fff6d23efdf 100644 --- a/app/presenters/clusterable_presenter.rb +++ b/app/presenters/clusterable_presenter.rb @@ -13,7 +13,8 @@ class ClusterablePresenter < Gitlab::View::Presenter::Delegated end def can_add_cluster? - can?(current_user, :add_cluster, clusterable) + can?(current_user, :add_cluster, clusterable) && + (has_no_clusters? || multiple_clusters_available?) end def can_create_cluster? @@ -63,4 +64,15 @@ class ClusterablePresenter < Gitlab::View::Presenter::Delegated def learn_more_link raise NotImplementedError end + + private + + # Overridden on EE module + def multiple_clusters_available? + false + end + + def has_no_clusters? + clusterable.clusters.empty? + end end diff --git a/app/serializers/current_board_entity.rb b/app/serializers/current_board_entity.rb new file mode 100644 index 00000000000..371151532f8 --- /dev/null +++ b/app/serializers/current_board_entity.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class CurrentBoardEntity < Grape::Entity + expose :id + expose :name +end diff --git a/app/serializers/current_board_serializer.rb b/app/serializers/current_board_serializer.rb new file mode 100644 index 00000000000..c58c77194f2 --- /dev/null +++ b/app/serializers/current_board_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class CurrentBoardSerializer < BaseSerializer + entity CurrentBoardEntity +end diff --git a/app/services/akismet_service.rb b/app/services/akismet_service.rb index 82ae66ab0f5..63be3c371ec 100644 --- a/app/services/akismet_service.rb +++ b/app/services/akismet_service.rb @@ -25,7 +25,7 @@ class AkismetService is_spam, is_blatant = akismet_client.check(options[:ip_address], options[:user_agent], params) is_spam || is_blatant rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") # rubocop:disable Gitlab/RailsLogger false end end @@ -63,7 +63,7 @@ class AkismetService akismet_client.public_send(type, options[:ip_address], options[:user_agent], params) # rubocop:disable GitlabSecurity/PublicSend true rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping!") + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping!") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/app/services/audit_event_service.rb b/app/services/audit_event_service.rb index 201048aaba5..73f3408a240 100644 --- a/app/services/audit_event_service.rb +++ b/app/services/audit_event_service.rb @@ -35,8 +35,12 @@ class AuditEventService @file_logger ||= Gitlab::AuditJsonLogger.build end + def formatted_details + @details.merge(@details.slice(:from, :to).transform_values(&:to_s)) + end + def log_security_event_to_file - file_logger.info(base_payload.merge(@details)) + file_logger.info(base_payload.merge(formatted_details)) end def log_security_event_to_database diff --git a/app/services/boards/issues/move_service.rb b/app/services/boards/issues/move_service.rb index e27d34dbcab..755d723b9a0 100644 --- a/app/services/boards/issues/move_service.rb +++ b/app/services/boards/issues/move_service.rb @@ -4,14 +4,37 @@ module Boards module Issues class MoveService < Boards::BaseService def execute(issue) - return false unless can?(current_user, :update_issue, issue) - return false if issue_params(issue).empty? + issue_modification_params = issue_params(issue) + return false if issue_modification_params.empty? - update(issue) + move_single_issue(issue, issue_modification_params) + end + + def execute_multiple(issues) + return false if issues.empty? + + last_inserted_issue_id = nil + issues.map do |issue| + issue_modification_params = issue_params(issue) + next if issue_modification_params.empty? + + if last_inserted_issue_id + issue_modification_params[:move_between_ids] = move_between_ids({ move_after_id: nil, move_before_id: last_inserted_issue_id }) + end + + last_inserted_issue_id = issue.id + move_single_issue(issue, issue_modification_params) + end.all? end private + def move_single_issue(issue, issue_modification_params) + return false unless can?(current_user, :update_issue, issue) + + update(issue, issue_modification_params) + end + def board @board ||= parent.boards.find(params[:board_id]) end @@ -33,8 +56,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def update(issue) - ::Issues::UpdateService.new(issue.project, current_user, issue_params(issue)).execute(issue) + def update(issue, issue_modification_params) + ::Issues::UpdateService.new(issue.project, current_user, issue_modification_params).execute(issue) end def issue_params(issue) @@ -48,6 +71,7 @@ module Boards ) end + move_between_ids = move_between_ids(params) if move_between_ids attrs[:move_between_ids] = move_between_ids attrs[:board_group_id] = board.group&.id @@ -78,8 +102,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def move_between_ids - ids = [params[:move_after_id], params[:move_before_id]] + def move_between_ids(move_params) + ids = [move_params[:move_after_id], move_params[:move_before_id]] .map(&:to_i) .map { |m| m.positive? ? m : nil } diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index a1dd00721b5..b5cfa1d019c 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -24,11 +24,11 @@ module Ci def archive_error(error, job) failed_archive_counter.increment - Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" + Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" # rubocop:disable Gitlab/RailsLogger Gitlab::Sentry .track_exception(error, - issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', + issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/51502', extra: { job_id: job.id }) end end diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 4a7ce00b8e2..aaf56048b5c 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -44,7 +44,7 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def stage_indexes_of_created_processables - created_processables.order(:stage_idx).pluck('distinct stage_idx') + created_processables.order(:stage_idx).pluck(Arel.sql('DISTINCT stage_idx')) end # rubocop: enable CodeReuse/ActiveRecord @@ -68,7 +68,7 @@ module Ci latest_statuses = pipeline.statuses.latest .group(:name) .having('count(*) > 1') - .pluck('max(id)', 'name') + .pluck(Arel.sql('MAX(id)'), 'name') # mark builds that are retried pipeline.statuses.latest diff --git a/app/services/clusters/create_service.rb b/app/services/clusters/create_service.rb index 886e484caaf..5fb5e15c32d 100644 --- a/app/services/clusters/create_service.rb +++ b/app/services/clusters/create_service.rb @@ -10,24 +10,27 @@ module Clusters def execute(access_token: nil) raise ArgumentError, 'Unknown clusterable provided' unless clusterable - raise ArgumentError, _('Instance does not support multiple Kubernetes clusters') unless can_create_cluster? cluster_params = params.merge(user: current_user).merge(clusterable_params) cluster_params[:provider_gcp_attributes].try do |provider| provider[:access_token] = access_token end - create_cluster(cluster_params).tap do |cluster| - ClusterProvisionWorker.perform_async(cluster.id) if cluster.persisted? + cluster = Clusters::Cluster.new(cluster_params) + + unless can_create_cluster? + cluster.errors.add(:base, _('Instance does not support multiple Kubernetes clusters')) end - end - private + return cluster if cluster.errors.present? - def create_cluster(cluster_params) - Clusters::Cluster.create(cluster_params) + cluster.tap do |cluster| + cluster.save && ClusterProvisionWorker.perform_async(cluster.id) + end end + private + def clusterable @clusterable ||= params.delete(:clusterable) end diff --git a/app/services/clusters/gcp/kubernetes.rb b/app/services/clusters/gcp/kubernetes.rb index 90ed529670c..85711764785 100644 --- a/app/services/clusters/gcp/kubernetes.rb +++ b/app/services/clusters/gcp/kubernetes.rb @@ -9,6 +9,8 @@ module Clusters GITLAB_CLUSTER_ROLE_BINDING_NAME = 'gitlab-admin' GITLAB_CLUSTER_ROLE_NAME = 'cluster-admin' PROJECT_CLUSTER_ROLE_NAME = 'edit' + GITLAB_KNATIVE_SERVING_ROLE_NAME = 'gitlab-knative-serving-role' + GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME = 'gitlab-knative-serving-rolebinding' end end end diff --git a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb index 49e766cbf13..7c5450dbcd6 100644 --- a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb +++ b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb @@ -41,7 +41,15 @@ module Clusters kubeclient.create_or_update_service_account(service_account_resource) kubeclient.create_or_update_secret(service_account_token_resource) - create_role_or_cluster_role_binding if rbac + + return unless rbac + + create_role_or_cluster_role_binding + + return unless namespace_creator + + create_or_update_knative_serving_role + create_or_update_knative_serving_role_binding end private @@ -63,6 +71,14 @@ module Clusters end end + def create_or_update_knative_serving_role + kubeclient.update_role(knative_serving_role_resource) + end + + def create_or_update_knative_serving_role_binding + kubeclient.update_role_binding(knative_serving_role_binding_resource) + end + def service_account_resource Gitlab::Kubernetes::ServiceAccount.new( service_account_name, @@ -92,6 +108,29 @@ module Clusters Gitlab::Kubernetes::RoleBinding.new( name: role_binding_name, role_name: Clusters::Gcp::Kubernetes::PROJECT_CLUSTER_ROLE_NAME, + role_kind: :ClusterRole, + namespace: service_account_namespace, + service_account_name: service_account_name + ).generate + end + + def knative_serving_role_resource + Gitlab::Kubernetes::Role.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + namespace: service_account_namespace, + rules: [{ + apiGroups: %w(serving.knative.dev), + resources: %w(configurations configurationgenerations routes revisions revisionuids autoscalers services), + verbs: %w(get list create update delete patch watch) + }] + ).generate + end + + def knative_serving_role_binding_resource + Gitlab::Kubernetes::RoleBinding.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, + role_name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + role_kind: :Role, namespace: service_account_namespace, service_account_name: service_account_name ).generate diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb index 2cb73555d85..0c5ecca3a50 100644 --- a/app/services/concerns/exclusive_lease_guard.rb +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -58,6 +58,6 @@ module ExclusiveLeaseGuard end def log_error(message, extra_args = {}) - Rails.logger.error(message) + Rails.logger.error(message) # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb index e9659f5489a..e78e5d5fc2c 100644 --- a/app/services/groups/create_service.rb +++ b/app/services/groups/create_service.rb @@ -27,8 +27,7 @@ module Groups @group.build_chat_team(name: response['name'], team_id: response['id']) end - @group.save - @group.add_owner(current_user) + @group.add_owner(current_user) if @group.save @group end diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index 654fe84e3dc..9e00cbbbc55 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -6,7 +6,7 @@ module Groups def async_execute job_id = GroupDestroyWorker.perform_async(group.id, current_user.id) - Rails.logger.info("User #{current_user.id} scheduled a deletion of group ID #{group.id} with job ID #{job_id}") + Rails.logger.info("User #{current_user.id} scheduled a deletion of group ID #{group.id} with job ID #{job_id}") # rubocop:disable Gitlab/RailsLogger end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/services/issuable/clone/content_rewriter.rb b/app/services/issuable/clone/content_rewriter.rb index 00d7078859d..f75b51c4be3 100644 --- a/app/services/issuable/clone/content_rewriter.rb +++ b/app/services/issuable/clone/content_rewriter.rb @@ -23,10 +23,14 @@ module Issuable end def rewrite_notes + new_discussion_ids = {} original_entity.notes_with_associations.find_each do |note| new_note = note.dup + new_discussion_ids[note.discussion_id] ||= Discussion.discussion_id(new_note) new_params = { - project: new_entity.project, noteable: new_entity, + project: new_entity.project, + noteable: new_entity, + discussion_id: new_discussion_ids[note.discussion_id], note: rewrite_content(new_note.note), note_html: nil, created_at: note.created_at, diff --git a/app/services/labels/create_service.rb b/app/services/labels/create_service.rb index db710bac900..c032985be42 100644 --- a/app/services/labels/create_service.rb +++ b/app/services/labels/create_service.rb @@ -20,7 +20,7 @@ module Labels label.save label else - Rails.logger.warn("target_params should contain :project or :group or :template, actual value: #{target_params}") + Rails.logger.warn("target_params should contain :project or :group or :template, actual value: #{target_params}") # rubocop:disable Gitlab/RailsLogger end end end diff --git a/app/services/merge_requests/merge_service.rb b/app/services/merge_requests/merge_service.rb index 3e0f5aa181c..6309052244d 100644 --- a/app/services/merge_requests/merge_service.rb +++ b/app/services/merge_requests/merge_service.rb @@ -113,12 +113,12 @@ module MergeRequests end def handle_merge_error(log_message:, save_message_on_model: false) - Rails.logger.error("MergeService ERROR: #{merge_request_info} - #{log_message}") + Rails.logger.error("MergeService ERROR: #{merge_request_info} - #{log_message}") # rubocop:disable Gitlab/RailsLogger @merge_request.update(merge_error: log_message) if save_message_on_model end def log_info(message) - @logger ||= Rails.logger + @logger ||= Rails.logger # rubocop:disable Gitlab/RailsLogger @logger.info("#{merge_request_info} - #{message}") end diff --git a/app/services/projects/after_import_service.rb b/app/services/projects/after_import_service.rb index bbdde4408d2..e30da0f26df 100644 --- a/app/services/projects/after_import_service.rb +++ b/app/services/projects/after_import_service.rb @@ -13,7 +13,7 @@ module Projects repository.delete_all_refs_except(RESERVED_REF_PREFIXES) end rescue Projects::HousekeepingService::LeaseTaken => e - Rails.logger.info( + Rails.logger.info( # rubocop:disable Gitlab/RailsLogger "Could not perform housekeeping for project #{@project.full_path} (#{@project.id}): #{e}") end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 9f335cceb67..89dc4375c63 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -151,7 +151,7 @@ module Projects log_message = message.dup log_message << " Project ID: #{@project.id}" if @project&.id - Rails.logger.error(log_message) + Rails.logger.error(log_message) # rubocop:disable Gitlab/RailsLogger if @project && @project.persisted? && @project.import_state @project.import_state.mark_as_failed(message) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index d8e670e40ce..b805a7f1211 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -18,7 +18,7 @@ module Projects schedule_stale_repos_removal job_id = ProjectDestroyWorker.perform_async(project.id, current_user.id, params) - Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.full_path} with job ID #{job_id}") + Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.full_path} with job ID #{job_id}") # rubocop:disable Gitlab/RailsLogger end def execute diff --git a/app/services/projects/hashed_storage/migrate_attachments_service.rb b/app/services/projects/hashed_storage/migrate_attachments_service.rb index 3d0b8f58612..affe6e5668d 100644 --- a/app/services/projects/hashed_storage/migrate_attachments_service.rb +++ b/app/services/projects/hashed_storage/migrate_attachments_service.rb @@ -5,7 +5,7 @@ module Projects class MigrateAttachmentsService < BaseAttachmentService def initialize(project, old_disk_path, logger: nil) @project = project - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @old_disk_path = old_disk_path @skipped = false end diff --git a/app/services/projects/hashed_storage/rollback_attachments_service.rb b/app/services/projects/hashed_storage/rollback_attachments_service.rb index 5c6b92f965c..fb09eaa4586 100644 --- a/app/services/projects/hashed_storage/rollback_attachments_service.rb +++ b/app/services/projects/hashed_storage/rollback_attachments_service.rb @@ -5,7 +5,7 @@ module Projects class RollbackAttachmentsService < BaseAttachmentService def initialize(project, logger: nil) @project = project - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @old_disk_path = project.disk_path end diff --git a/app/services/projects/hashed_storage/rollback_service.rb b/app/services/projects/hashed_storage/rollback_service.rb index 25767f5de5e..ee41aae64a5 100644 --- a/app/services/projects/hashed_storage/rollback_service.rb +++ b/app/services/projects/hashed_storage/rollback_service.rb @@ -8,7 +8,7 @@ module Projects def initialize(project, old_disk_path, logger: nil) @project = project @old_disk_path = old_disk_path - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def execute diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index e3491282a8a..9c6d7ef41f6 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -62,7 +62,7 @@ module Projects end def cleanup_and_notify_error - Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") + Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger FileUtils.rm_rf(@shared.export_path) @@ -76,7 +76,7 @@ module Projects end def notify_success - Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported") + Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported") # rubocop:disable Gitlab/RailsLogger end def notify_error diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index a25c985585b..64f9b611c40 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -15,7 +15,7 @@ module Projects def propagate return unless @template.active? - Rails.logger.info("Propagating services for template #{@template.id}") + Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger propagate_projects_with_template end diff --git a/app/services/projects/update_statistics_service.rb b/app/services/projects/update_statistics_service.rb index 28677a398f3..cc6ffa9eafc 100644 --- a/app/services/projects/update_statistics_service.rb +++ b/app/services/projects/update_statistics_service.rb @@ -5,7 +5,7 @@ module Projects def execute return unless project - Rails.logger.info("Updating statistics for project #{project.id}") + Rails.logger.info("Updating statistics for project #{project.id}") # rubocop:disable Gitlab/RailsLogger project.statistics.refresh!(only: statistics.map(&:to_sym)) end diff --git a/app/services/submit_usage_ping_service.rb b/app/services/submit_usage_ping_service.rb index 62222d3fd2a..4f10f220298 100644 --- a/app/services/submit_usage_ping_service.rb +++ b/app/services/submit_usage_ping_service.rb @@ -28,7 +28,7 @@ class SubmitUsagePingService true rescue Gitlab::HTTP::Error => e - Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" + Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" # rubocop:disable Gitlab/RailsLogger false end diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 1fee8bfcd31..6d675c026bb 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -53,7 +53,7 @@ class WebHookService error_message: e.to_s ) - Rails.logger.error("WebHook Error => #{e}") + Rails.logger.error("WebHook Error => #{e}") # rubocop:disable Gitlab/RailsLogger { status: :error, diff --git a/app/uploaders/file_mover.rb b/app/uploaders/file_mover.rb index 12be1e2bb22..7c7953c8a0e 100644 --- a/app/uploaders/file_mover.rb +++ b/app/uploaders/file_mover.rb @@ -98,7 +98,7 @@ class FileMover end def revert - Rails.logger.warn("Markdown not updated, file move reverted for #{to_model}") + Rails.logger.warn("Markdown not updated, file move reverted for #{to_model}") # rubocop:disable Gitlab/RailsLogger if temp_file_uploader.file_storage? FileUtils.move(file_path, temp_file_path) diff --git a/app/validators/namespace_name_validator.rb b/app/validators/namespace_name_validator.rb deleted file mode 100644 index fb1c241037c..00000000000 --- a/app/validators/namespace_name_validator.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -# NamespaceNameValidator -# -# Custom validator for GitLab namespace name strings. -class NamespaceNameValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless value =~ Gitlab::Regex.namespace_name_regex - record.errors.add(attribute, Gitlab::Regex.namespace_name_regex_message) - end - end -end diff --git a/app/views/ci/status/_icon.html.haml b/app/views/ci/status/_icon.html.haml index f38bdb2e5ed..1249b98221f 100644 --- a/app/views/ci/status/_icon.html.haml +++ b/app/views/ci/status/_icon.html.haml @@ -1,9 +1,10 @@ -- status = local_assigns.fetch(:status) -- size = local_assigns.fetch(:size, 16) -- type = local_assigns.fetch(:type, 'pipeline') -- tooltip_placement = local_assigns.fetch(:tooltip_placement, "left") -- path = local_assigns.fetch(:path, status.has_details? ? status.details_path : nil) -- css_classes = "ci-status-link ci-status-icon ci-status-icon-#{status.group} has-tooltip" +- status = local_assigns.fetch(:status) +- size = local_assigns.fetch(:size, 16) +- type = local_assigns.fetch(:type, 'pipeline') +- tooltip_placement = local_assigns.fetch(:tooltip_placement, "left") +- path = local_assigns.fetch(:path, status.has_details? ? status.details_path : nil) +- option_css_classes = local_assigns.fetch(:option_css_classes, '') +- css_classes = "ci-status-link ci-status-icon ci-status-icon-#{status.group} has-tooltip #{option_css_classes}" - title = s_("PipelineStatusTooltip|Pipeline: %{ci_status}") % {ci_status: status.label} - if type == 'commit' - title = s_("PipelineStatusTooltip|Commit: %{ci_status}") % {ci_status: status.label} diff --git a/app/views/devise/sessions/_new_base.html.haml b/app/views/devise/sessions/_new_base.html.haml index 7dacd0b1d72..2f10f08c839 100644 --- a/app/views/devise/sessions/_new_base.html.haml +++ b/app/views/devise/sessions/_new_base.html.haml @@ -1,10 +1,10 @@ = form_for(resource, as: resource_name, url: session_path(resource_name), html: { class: 'new_user gl-show-field-errors', 'aria-live' => 'assertive'}) do |f| .form-group = f.label "Username or email", for: "user_login", class: 'label-bold' - = f.text_field :login, class: "form-control top qa-login-field", autofocus: "autofocus", autocapitalize: "off", autocorrect: "off", required: true, title: "This field is required." + = f.text_field :login, class: "form-control top", autofocus: "autofocus", autocapitalize: "off", autocorrect: "off", required: true, title: "This field is required.", data: { qa_selector: 'login_field' } .form-group = f.label :password, class: 'label-bold' - = f.password_field :password, class: "form-control bottom qa-password-field", required: true, title: "This field is required." + = f.password_field :password, class: "form-control bottom", required: true, title: "This field is required.", data: { qa_selector: 'password_field' } - if devise_mapping.rememberable? .remember-me %label{ for: "user_remember_me" } @@ -17,4 +17,4 @@ = recaptcha_tags .submit-container.move-submit-down - = f.submit "Sign in", class: "btn btn-success qa-sign-in-button" + = f.submit "Sign in", class: "btn btn-success", data: { qa_selector: 'sign_in_button' } diff --git a/app/views/layouts/devise.html.haml b/app/views/layouts/devise.html.haml index ff3410f6268..e9a4a068599 100644 --- a/app/views/layouts/devise.html.haml +++ b/app/views/layouts/devise.html.haml @@ -1,7 +1,7 @@ !!! 5 %html.devise-layout-html{ class: system_message_class } = render "layouts/head" - %body.ui-indigo.login-page.application.navless.qa-login-page{ data: { page: body_data_page } } + %body.ui-indigo.login-page.application.navless{ data: { page: body_data_page, qa_selector: 'login_page' } } = header_message .page-wrap = render "layouts/header/empty" diff --git a/app/views/layouts/header/_current_user_dropdown.html.haml b/app/views/layouts/header/_current_user_dropdown.html.haml index 4f3e4031fe3..45fc39dbbdb 100644 --- a/app/views/layouts/header/_current_user_dropdown.html.haml +++ b/app/views/layouts/header/_current_user_dropdown.html.haml @@ -24,4 +24,4 @@ - if current_user_menu?(:sign_out) %li.divider %li - = link_to _("Sign out"), destroy_user_session_path, class: "sign-out-link" + = link_to _("Sign out"), destroy_user_session_path, class: "sign-out-link qa-sign-out-link" diff --git a/app/views/layouts/header/_default.html.haml b/app/views/layouts/header/_default.html.haml index f9ee6f42e23..fc2dea25c77 100644 --- a/app/views/layouts/header/_default.html.haml +++ b/app/views/layouts/header/_default.html.haml @@ -5,7 +5,7 @@ - else - search_path_url = search_path -%header.navbar.navbar-gitlab.qa-navbar.navbar-expand-sm.js-navbar +%header.navbar.navbar-gitlab.navbar-expand-sm.js-navbar{ data: { qa_selector: 'navbar' } } %a.sr-only.gl-accessibility{ href: "#content-body", tabindex: "1" } Skip to content .container-fluid .header-content @@ -64,7 +64,7 @@ .dropdown-menu.dropdown-menu-right = render 'layouts/header/help_dropdown' - if header_link?(:user_dropdown) - %li.nav-item.header-user.dropdown{ data: { track_label: "profile_dropdown", track_event: "click_dropdown" } } + %li.nav-item.header-user.dropdown{ data: { track_label: "profile_dropdown", track_event: "click_dropdown", qa_selector: 'user_menu' } } = link_to current_user, class: user_dropdown_class, data: { toggle: "dropdown" } do = image_tag avatar_icon_for_user(current_user, 23), width: 23, height: 23, class: "header-user-avatar qa-user-avatar" = sprite_icon('angle-down', css_class: 'caret-down') diff --git a/app/views/projects/environments/show.html.haml b/app/views/projects/environments/show.html.haml index c13a47b0b09..6100fd3ad37 100644 --- a/app/views/projects/environments/show.html.haml +++ b/app/views/projects/environments/show.html.haml @@ -3,6 +3,9 @@ - breadcrumb_title @environment.name - page_title _("Environments") +- content_for :page_specific_javascripts do + = stylesheet_link_tag 'page_bundles/xterm' + %div{ class: container_class } - if can?(current_user, :stop_environment, @environment) #stop-environment-modal.modal.fade{ tabindex: -1 } diff --git a/app/views/projects/issues/import_csv/_button.html.haml b/app/views/projects/issues/import_csv/_button.html.haml index acc2c50294f..8442a53ed61 100644 --- a/app/views/projects/issues/import_csv/_button.html.haml +++ b/app/views/projects/issues/import_csv/_button.html.haml @@ -1,6 +1,6 @@ - type = local_assigns.fetch(:type, :icon) -%button.csv-import-button.btn{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon), +%button.csv-import-button.btn.btn-svg{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon), data: { toggle: 'modal', target: '.issues-import-modal' } } - if type == :icon = sprite_icon('upload') diff --git a/app/views/projects/merge_requests/_merge_request.html.haml b/app/views/projects/merge_requests/_merge_request.html.haml index eb516684e52..dee3931ff04 100644 --- a/app/views/projects/merge_requests/_merge_request.html.haml +++ b/app/views/projects/merge_requests/_merge_request.html.haml @@ -38,7 +38,7 @@ = link_to_label(label, type: :merge_request, css_class: 'label-link') .issuable-meta - %ul.controls + %ul.controls.d-flex.align-items-end - if merge_request.merged? %li.issuable-status.d-none.d-sm-inline-block MERGED @@ -47,14 +47,14 @@ = icon('ban') CLOSED - if can?(current_user, :read_pipeline, merge_request.head_pipeline) - %li.issuable-pipeline-status.d-none.d-sm-inline-block - = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user) + %li.issuable-pipeline-status.d-none.d-sm-flex + = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user), option_css_classes: 'd-flex' - if merge_request.open? && merge_request.broken? - %li.issuable-pipeline-broken.d-none.d-sm-inline-block + %li.issuable-pipeline-broken.d-none.d-sm-flex = link_to merge_request_path(merge_request), class: "has-tooltip", title: _('Cannot be merged automatically') do = icon('exclamation-triangle') - if merge_request.assignees.any? - %li + %li.d-flex = render 'shared/issuable/assignees', project: merge_request.project, issuable: merge_request = render_if_exists 'projects/merge_requests/approvals_count', merge_request: merge_request diff --git a/app/views/projects/settings/operations/_external_dashboard.html.haml b/app/views/projects/settings/operations/_external_dashboard.html.haml index a124283921d..08d50a336fd 100644 --- a/app/views/projects/settings/operations/_external_dashboard.html.haml +++ b/app/views/projects/settings/operations/_external_dashboard.html.haml @@ -1,3 +1,3 @@ .js-operation-settings{ data: { operations_settings_endpoint: project_settings_operations_path(@project), external_dashboard: { url: metrics_external_dashboard_url, - help_page_path: help_page_path('user/project/operations/link_to_external_dashboard') } } } + help_page_path: help_page_path('user/project/operations/linking_to_an_external_dashboard') } } } diff --git a/app/views/projects/tree/_tree_content.html.haml b/app/views/projects/tree/_tree_content.html.haml index 889a13339fd..cb459b031fc 100644 --- a/app/views/projects/tree/_tree_content.html.haml +++ b/app/views/projects/tree/_tree_content.html.haml @@ -1,4 +1,4 @@ -.tree-content-holder.js-tree-content{ 'data-logs-path': @logs_path } +.tree-content-holder.js-tree-content{ data: tree_content_data(@logs_path, @project, @path) } .table-holder.bordered-box %table.table#tree-slider{ class: "table_#{@hex_path} tree-table qa-file-tree" } %thead diff --git a/app/views/shared/issuable/_feed_buttons.html.haml b/app/views/shared/issuable/_feed_buttons.html.haml index 83f60fa6fe2..c9506a3295c 100644 --- a/app/views/shared/issuable/_feed_buttons.html.haml +++ b/app/views/shared/issuable/_feed_buttons.html.haml @@ -1,4 +1,4 @@ -= link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to RSS feed') do - = icon('rss') -= link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do - = custom_icon('icon_calendar') += link_to safe_params.merge(rss_url_options), class: 'btn btn-svg has-tooltip js-rss-button', data: { container: 'body' }, title: _('Subscribe to RSS feed') do + = sprite_icon('rss') += link_to safe_params.merge(calendar_url_options), class: 'btn btn-svg has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do + = sprite_icon('calendar') diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml index 90fb067e75d..71bd9320593 100644 --- a/app/views/shared/projects/_project.html.haml +++ b/app/views/shared/projects/_project.html.haml @@ -58,7 +58,7 @@ .description.d-none.d-sm-block.append-right-default = markdown_field(project, :description) - .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0{ class: css_controls_class } + .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0.text-secondary{ class: css_controls_class } .icon-container.d-flex.align-items-center - if project.archived %span.d-flex.icon-wrapper.badge.badge-warning archived diff --git a/app/workers/concerns/new_issuable.rb b/app/workers/concerns/new_issuable.rb index a89451a4475..22ba7c97309 100644 --- a/app/workers/concerns/new_issuable.rb +++ b/app/workers/concerns/new_issuable.rb @@ -27,6 +27,6 @@ module NewIssuable # rubocop: enable CodeReuse/ActiveRecord def log_error(record_class, record_id) - Rails.logger.error("#{self.class}: couldn't find #{record_class} with ID=#{record_id}, skipping job") + Rails.logger.error("#{self.class}: couldn't find #{record_class} with ID=#{record_id}, skipping job") # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/workers/create_gpg_signature_worker.rb b/app/workers/create_gpg_signature_worker.rb index 7fac7822cf7..e3fb5d479ae 100644 --- a/app/workers/create_gpg_signature_worker.rb +++ b/app/workers/create_gpg_signature_worker.rb @@ -22,7 +22,7 @@ class CreateGpgSignatureWorker commits.each do |commit| Gitlab::Gpg::Commit.new(commit).signature rescue => e - Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") + Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") # rubocop:disable Gitlab/RailsLogger end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb index 4d0295f8d2e..efa8794b214 100644 --- a/app/workers/delete_user_worker.rb +++ b/app/workers/delete_user_worker.rb @@ -9,6 +9,6 @@ class DeleteUserWorker Users::DestroyService.new(current_user).execute(delete_user, options.symbolize_keys) rescue Gitlab::Access::AccessDeniedError => e - Rails.logger.warn("User could not be destroyed: #{e}") + Rails.logger.warn("User could not be destroyed: #{e}") # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index c4bcda2da16..e70bf17d5a9 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -16,7 +16,7 @@ class EmailReceiverWorker private def handle_failure(raw, error) - Rails.logger.warn("Email can not be processed: #{error}\n\n#{raw}") + Rails.logger.warn("Email can not be processed: #{error}\n\n#{raw}") # rubocop:disable Gitlab/RailsLogger return unless raw.present? diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb index 251e95c68d5..6f0e0fd33f7 100644 --- a/app/workers/expire_build_artifacts_worker.rb +++ b/app/workers/expire_build_artifacts_worker.rb @@ -18,7 +18,7 @@ class ExpireBuildArtifactsWorker # rubocop: disable CodeReuse/ActiveRecord def perform_legacy_artifacts_removal - Rails.logger.info 'Scheduling removal of build artifacts' + Rails.logger.info 'Scheduling removal of build artifacts' # rubocop:disable Gitlab/RailsLogger build_ids = Ci::Build.with_expired_artifacts.pluck(:id) build_ids = build_ids.map { |build_id| [build_id] } diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb index 94426dcf921..71e61dcb878 100644 --- a/app/workers/expire_build_instance_artifacts_worker.rb +++ b/app/workers/expire_build_instance_artifacts_worker.rb @@ -12,7 +12,7 @@ class ExpireBuildInstanceArtifactsWorker return unless build&.project && !build.project.pending_delete - Rails.logger.info "Removing artifacts for build #{build.id}..." + Rails.logger.info "Removing artifacts for build #{build.id}..." # rubocop:disable Gitlab/RailsLogger build.erase_erasable_artifacts! end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index 98f9f45e608..1d1ea926c21 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -11,7 +11,7 @@ class NewNoteWorker NotificationService.new.new_note(note) unless skip_notification?(note) Notes::PostProcessService.new(note).execute else - Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") + Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb index 12400d4e025..55ac7cd9b3c 100644 --- a/app/workers/object_storage/migrate_uploads_worker.rb +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -37,6 +37,7 @@ module ObjectStorage end end + # rubocop:disable Gitlab/RailsLogger def report!(results) success, failures = results.partition(&:success?) @@ -45,6 +46,7 @@ module ObjectStorage raise MigrationFailures.new(failures.map(&:error)) if failures.any? end + # rubocop:enable Gitlab/RailsLogger def header(success, failures) _("Migrated %{success_count}/%{total_count} files.") % { success_count: success.count, total_count: success.count + failures.count } @@ -98,7 +100,7 @@ module ObjectStorage report!(results) rescue SanityCheckError => e # do not retry: the job is insane - Rails.logger.warn "#{self.class}: Sanity check error (#{e.message})" + Rails.logger.warn "#{self.class}: Sanity check error (#{e.message})" # rubocop:disable Gitlab/RailsLogger end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index a9b88a133be..35e9c58eb13 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -35,7 +35,7 @@ class RepositoryForkWorker def start_fork(project) return true if start(project.import_state) - Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while forking.") + Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while forking.") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 59691f48a39..dff9c8f50bf 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -36,7 +36,7 @@ class RepositoryImportWorker def start_import return true if start(project.import_state) - Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.") + Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.") # rubocop:disable Gitlab/RailsLogger false end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb index c0bae08ba85..03a7ff2cd7a 100644 --- a/app/workers/repository_update_remote_mirror_worker.rb +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -45,6 +45,6 @@ class RepositoryUpdateRemoteMirrorWorker def fail_remote_mirror(remote_mirror, message) remote_mirror.mark_as_failed(message) - Rails.logger.error(message) + Rails.logger.error(message) # rubocop:disable Gitlab/RailsLogger end end diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb index 43e0b9db22f..351850e53cb 100644 --- a/app/workers/run_pipeline_schedule_worker.rb +++ b/app/workers/run_pipeline_schedule_worker.rb @@ -30,6 +30,7 @@ class RunPipelineScheduleWorker private + # rubocop:disable Gitlab/RailsLogger def error(schedule, error) failed_creation_counter.increment @@ -41,6 +42,7 @@ class RunPipelineScheduleWorker issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/41231', extra: { schedule_id: schedule.id }) end + # rubocop:enable Gitlab/RailsLogger def failed_creation_counter @failed_creation_counter ||= diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb index 25809f68080..30fba038937 100644 --- a/app/workers/stuck_ci_jobs_worker.rb +++ b/app/workers/stuck_ci_jobs_worker.rb @@ -14,7 +14,7 @@ class StuckCiJobsWorker def perform return unless try_obtain_lease - Rails.logger.info "#{self.class}: Cleaning stuck builds" + Rails.logger.info "#{self.class}: Cleaning stuck builds" # rubocop:disable Gitlab/RailsLogger drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure @@ -66,7 +66,7 @@ class StuckCiJobsWorker # rubocop: enable CodeReuse/ActiveRecord def drop_build(type, build, status, timeout, reason) - Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" + Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" # rubocop:disable Gitlab/RailsLogger Gitlab::OptimisticLocking.retry_lock(build, 3) do |b| b.drop(reason) end diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index c8a186ba4ce..a9ff5b22b25 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -38,7 +38,7 @@ class StuckImportJobsWorker completed_import_states = enqueued_import_states_with_jid.where(id: completed_import_state_ids) completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ') - Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}") + Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}") # rubocop:disable Gitlab/RailsLogger completed_import_states.each do |import_state| import_state.mark_as_failed(error_message) diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb index f34ed6c4844..e840ae47421 100644 --- a/app/workers/stuck_merge_jobs_worker.rb +++ b/app/workers/stuck_merge_jobs_worker.rb @@ -5,7 +5,7 @@ class StuckMergeJobsWorker include CronjobQueue def self.logger - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb index 3297a1fe3d0..55b599ba38f 100644 --- a/app/workers/trending_projects_worker.rb +++ b/app/workers/trending_projects_worker.rb @@ -5,7 +5,7 @@ class TrendingProjectsWorker include CronjobQueue def perform - Rails.logger.info('Refreshing trending projects') + Rails.logger.info('Refreshing trending projects') # rubocop:disable Gitlab/RailsLogger TrendingProject.refresh! end diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb index c7213df652a..6c0e472e05a 100644 --- a/app/workers/update_merge_requests_worker.rb +++ b/app/workers/update_merge_requests_worker.rb @@ -27,7 +27,7 @@ class UpdateMergeRequestsWorker "ref=#{ref}" ].join(',') - Rails.logger.info("UpdateMergeRequestsWorker#perform #{args_log}") if time.real > LOG_TIME_THRESHOLD + Rails.logger.info("UpdateMergeRequestsWorker#perform #{args_log}") if time.real > LOG_TIME_THRESHOLD # rubocop:disable Gitlab/RailsLogger end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb index 2a0536106d7..834dcaa435d 100644 --- a/app/workers/upload_checksum_worker.rb +++ b/app/workers/upload_checksum_worker.rb @@ -8,6 +8,6 @@ class UploadChecksumWorker upload.calculate_checksum! upload.save! rescue ActiveRecord::RecordNotFound - Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") + Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") # rubocop:disable Gitlab/RailsLogger end end diff --git a/bin/bundle b/bin/bundle index 66e9889e8b4..f19acf5b5cc 100755 --- a/bin/bundle +++ b/bin/bundle @@ -1,3 +1,3 @@ #!/usr/bin/env ruby -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) load Gem.bin_path('bundler', 'bundle') diff --git a/bin/setup b/bin/setup index 883825bc0a2..94fd4d79775 100755 --- a/bin/setup +++ b/bin/setup @@ -1,33 +1,36 @@ #!/usr/bin/env ruby - -require "pathname" +require 'fileutils' +include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path("../../", __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end -Dir.chdir APP_ROOT do - # This script is a starting point to set up your application. - # Add necessary setup steps to this file: +chdir APP_ROOT do + # This script is a starting point to setup your application. + # Add necessary setup steps to this file. + + puts '== Installing dependencies ==' + system! 'gem install bundler --conservative' + system('bundle check') || system!('bundle install') - puts "== Installing dependencies ==" - system! "gem install bundler --conservative" - system("bundle check") || system!("bundle install") + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') # puts "\n== Copying sample files ==" - # unless File.exist?("config/database.yml") - # cp "config/database.yml.sample", "config/database.yml" + # unless File.exist?('config/database.yml') + # cp 'config/database.yml.sample', 'config/database.yml' # end puts "\n== Preparing database ==" - system! "bin/rails db:setup" + system! 'bin/rails db:setup' puts "\n== Removing old logs and tempfiles ==" - system! "bin/rails log:clear tmp:clear" + system! 'bin/rails log:clear tmp:clear' puts "\n== Restarting application server ==" - system! "bin/rails restart" + system! 'bin/rails restart' end diff --git a/bin/update b/bin/update index a8e4462f203..58bfaed518c 100755 --- a/bin/update +++ b/bin/update @@ -1,10 +1,9 @@ #!/usr/bin/env ruby -require 'pathname' require 'fileutils' include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") @@ -18,6 +17,9 @@ chdir APP_ROOT do system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') + puts "\n== Updating database ==" system! 'bin/rails db:migrate' @@ -1,63 +1,21 @@ #!/bin/sh +set -e + cd $(dirname $0)/.. app_root=$(pwd) -# Switch to experimental PUMA configuration -if [ -n "${EXPERIMENTAL_PUMA}" ]; then - exec bin/web_puma "$@" -fi - -unicorn_pidfile="$app_root/tmp/pids/unicorn.pid" -unicorn_config="$app_root/config/unicorn.rb" -unicorn_cmd="bundle exec unicorn_rails -c $unicorn_config -E $RAILS_ENV" - -get_unicorn_pid() -{ - local pid=$(cat $unicorn_pidfile) - if [ -z "$pid" ] ; then - echo "Could not find a PID in $unicorn_pidfile" - exit 1 - fi - unicorn_pid=$pid -} - -start() -{ - exec $unicorn_cmd -D -} - -start_foreground() -{ - exec $unicorn_cmd -} - -stop() -{ - get_unicorn_pid - kill -QUIT $unicorn_pid -} +case "$USE_WEB_SERVER" in + puma|"") # and the "" defines default + exec bin/web_puma "$@" + ;; -reload() -{ - get_unicorn_pid - kill -USR2 $unicorn_pid -} + unicorn) + exec bin/web_unicorn "$@" + ;; -case "$1" in - start) - start - ;; - start_foreground) - start_foreground - ;; - stop) - stop - ;; - reload) - reload - ;; - *) - echo "Usage: RAILS_ENV=your_env $0 {start|stop|reload}" - ;; + *) + echo "Unkown web server used by USE_WEB_SERVER: $USE_WEB_SERVER." + exit 1 + ;; esac diff --git a/bin/web_unicorn b/bin/web_unicorn new file mode 100755 index 00000000000..ecd0bbd10b0 --- /dev/null +++ b/bin/web_unicorn @@ -0,0 +1,58 @@ +#!/bin/sh + +cd $(dirname $0)/.. +app_root=$(pwd) + +unicorn_pidfile="$app_root/tmp/pids/unicorn.pid" +unicorn_config="$app_root/config/unicorn.rb" +unicorn_cmd="bundle exec unicorn_rails -c $unicorn_config -E $RAILS_ENV" + +get_unicorn_pid() +{ + local pid=$(cat $unicorn_pidfile) + if [ -z "$pid" ] ; then + echo "Could not find a PID in $unicorn_pidfile" + exit 1 + fi + unicorn_pid=$pid +} + +start() +{ + exec $unicorn_cmd -D +} + +start_foreground() +{ + exec $unicorn_cmd +} + +stop() +{ + get_unicorn_pid + kill -QUIT $unicorn_pid +} + +reload() +{ + get_unicorn_pid + kill -USR2 $unicorn_pid +} + +case "$1" in + start) + start + ;; + start_foreground) + start_foreground + ;; + stop) + stop + ;; + reload) + reload + ;; + *) + echo "Usage: RAILS_ENV=your_env $0 {start|stop|reload}" + ;; +esac diff --git a/bin/yarn b/bin/yarn new file mode 100755 index 00000000000..460dd565b4a --- /dev/null +++ b/bin/yarn @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +APP_ROOT = File.expand_path('..', __dir__) +Dir.chdir(APP_ROOT) do + begin + exec "yarnpkg", *ARGV + rescue Errno::ENOENT + $stderr.puts "Yarn executable was not detected in the system." + $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" + exit 1 + end +end diff --git a/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml b/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml new file mode 100644 index 00000000000..ae50350f70d --- /dev/null +++ b/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml @@ -0,0 +1,5 @@ +--- +title: Add endpoint to move multiple issues in boards +merge_request: 30216 +author: +type: added diff --git a/changelogs/unreleased/55623-group-cluster-apis.yml b/changelogs/unreleased/55623-group-cluster-apis.yml new file mode 100644 index 00000000000..fe987ef4a82 --- /dev/null +++ b/changelogs/unreleased/55623-group-cluster-apis.yml @@ -0,0 +1,5 @@ +--- +title: Add API for CRUD group clusters +merge_request: 30213 +author: +type: added diff --git a/changelogs/unreleased/61145-fix-button-dimensions.yml b/changelogs/unreleased/61145-fix-button-dimensions.yml new file mode 100644 index 00000000000..8f209ceaa8e --- /dev/null +++ b/changelogs/unreleased/61145-fix-button-dimensions.yml @@ -0,0 +1,5 @@ +--- +title: Updating button dimensions according to design spec +merge_request: 28545 +author: +type: fixed diff --git a/changelogs/unreleased/62088-search-back.yml b/changelogs/unreleased/62088-search-back.yml new file mode 100644 index 00000000000..4758e927880 --- /dev/null +++ b/changelogs/unreleased/62088-search-back.yml @@ -0,0 +1,5 @@ +--- +title: Fixed back navigation for projects filter +merge_request: 30373 +author: +type: fixed diff --git a/changelogs/unreleased/63298-prevent-excessive-sanitization-asciidoc.yml b/changelogs/unreleased/63298-prevent-excessive-sanitization-asciidoc.yml new file mode 100644 index 00000000000..cd8206cdb99 --- /dev/null +++ b/changelogs/unreleased/63298-prevent-excessive-sanitization-asciidoc.yml @@ -0,0 +1,5 @@ +--- +title: "Prevent excessive sanitization of AsciiDoc ouptut" +merge_request: 30290 +author: Guillaume Grossetie +type: added
\ No newline at end of file diff --git a/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml b/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml new file mode 100644 index 00000000000..ead79be2505 --- /dev/null +++ b/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml @@ -0,0 +1,5 @@ +--- +title: Display the correct amount of projects being migrated/rolled-back to Hashed Storage when specifying ranges +merge_request: 29996 +author: +type: fixed diff --git a/changelogs/unreleased/63691-fix-doc-link.yml b/changelogs/unreleased/63691-fix-doc-link.yml new file mode 100644 index 00000000000..e63756e8227 --- /dev/null +++ b/changelogs/unreleased/63691-fix-doc-link.yml @@ -0,0 +1,5 @@ +--- +title: Correct link to docs for External Dashboard +merge_request: 30019 +author: +type: fixed diff --git a/changelogs/unreleased/64091-fix-broken-terminal.yml b/changelogs/unreleased/64091-fix-broken-terminal.yml new file mode 100644 index 00000000000..156f6de5008 --- /dev/null +++ b/changelogs/unreleased/64091-fix-broken-terminal.yml @@ -0,0 +1,5 @@ +--- +title: Fix environments broken terminal +merge_request: 30401 +author: +type: fixed diff --git a/changelogs/unreleased/64161-gitlab-fqdn.yml b/changelogs/unreleased/64161-gitlab-fqdn.yml new file mode 100644 index 00000000000..2946be37caa --- /dev/null +++ b/changelogs/unreleased/64161-gitlab-fqdn.yml @@ -0,0 +1,5 @@ +--- +title: Add CI variable to provide GitLab HOST +merge_request: 30417 +author: +type: added diff --git a/changelogs/unreleased/64314-ci-icon.yml b/changelogs/unreleased/64314-ci-icon.yml new file mode 100644 index 00000000000..8a550b6fa5b --- /dev/null +++ b/changelogs/unreleased/64314-ci-icon.yml @@ -0,0 +1,5 @@ +--- +title: Aligns CI icon in Merge Request dashboard +merge_request: 30558 +author: +type: fixed diff --git a/changelogs/unreleased/backstage-gb-improve-performance-environment-statuses-endpoint.yml b/changelogs/unreleased/backstage-gb-improve-performance-environment-statuses-endpoint.yml new file mode 100644 index 00000000000..f614e076268 --- /dev/null +++ b/changelogs/unreleased/backstage-gb-improve-performance-environment-statuses-endpoint.yml @@ -0,0 +1,5 @@ +--- +title: Improve performance of fetching environments statuses +merge_request: 30560 +author: +type: performance diff --git a/changelogs/unreleased/bvl-rename-routes-after-user-rename.yml b/changelogs/unreleased/bvl-rename-routes-after-user-rename.yml new file mode 100644 index 00000000000..8b0a01dea53 --- /dev/null +++ b/changelogs/unreleased/bvl-rename-routes-after-user-rename.yml @@ -0,0 +1,5 @@ +--- +title: Update a user's routes after updating their name +merge_request: 23272 +author: +type: fixed diff --git a/changelogs/unreleased/fe-delete-old-boardservice-on-component-modal-footer.yml b/changelogs/unreleased/fe-delete-old-boardservice-on-component-modal-footer.yml new file mode 100644 index 00000000000..4392b72443b --- /dev/null +++ b/changelogs/unreleased/fe-delete-old-boardservice-on-component-modal-footer.yml @@ -0,0 +1,6 @@ +--- +title: change the use of boardService in favor of boardsStore on footer for the board + component +merge_request: 30616 +author: eduarmreyes +type: other diff --git a/changelogs/unreleased/gitaly-version-v1-53-0.yml b/changelogs/unreleased/gitaly-version-v1-53-0.yml new file mode 100644 index 00000000000..7d3e5ce3cb3 --- /dev/null +++ b/changelogs/unreleased/gitaly-version-v1-53-0.yml @@ -0,0 +1,5 @@ +--- +title: Upgrade to Gitaly v1.53.0 +merge_request: 30614 +author: +type: changed diff --git a/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml b/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml new file mode 100644 index 00000000000..958334cc28e --- /dev/null +++ b/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml @@ -0,0 +1,5 @@ +--- +title: Create Knative role and binding with service account +merge_request: 30235 +author: +type: changed diff --git a/changelogs/unreleased/id-clean-up-mr-assignees.yml b/changelogs/unreleased/id-clean-up-mr-assignees.yml new file mode 100644 index 00000000000..7ff03c9f00b --- /dev/null +++ b/changelogs/unreleased/id-clean-up-mr-assignees.yml @@ -0,0 +1,5 @@ +--- +title: Add cleanup migration for MR's multiple assignees +merge_request: 30261 +author: +type: changed diff --git a/changelogs/unreleased/mw-project-list-color-fix.yml b/changelogs/unreleased/mw-project-list-color-fix.yml new file mode 100644 index 00000000000..6f8b2742ec6 --- /dev/null +++ b/changelogs/unreleased/mw-project-list-color-fix.yml @@ -0,0 +1,5 @@ +--- +title: Add text-secondary to controls in project list +merge_request: 30567 +author: +type: fixed diff --git a/changelogs/unreleased/registry-fix-multi-delete-modal.yml b/changelogs/unreleased/registry-fix-multi-delete-modal.yml new file mode 100644 index 00000000000..94a2df7a7e7 --- /dev/null +++ b/changelogs/unreleased/registry-fix-multi-delete-modal.yml @@ -0,0 +1,5 @@ +--- +title: Prevent multiple confirmation modals from opening when deleting a repository +merge_request: 30532 +author: +type: fixed diff --git a/changelogs/unreleased/sh-audit-event-json-log-format-from-and-to.yml b/changelogs/unreleased/sh-audit-event-json-log-format-from-and-to.yml new file mode 100644 index 00000000000..5bd21d7ef2b --- /dev/null +++ b/changelogs/unreleased/sh-audit-event-json-log-format-from-and-to.yml @@ -0,0 +1,5 @@ +--- +title: Format `from` and `to` fields in JSON audit log +merge_request: 30333 +author: +type: changed diff --git a/changelogs/unreleased/update-pages-to-1-7-0.yml b/changelogs/unreleased/update-pages-to-1-7-0.yml new file mode 100644 index 00000000000..d41346f021c --- /dev/null +++ b/changelogs/unreleased/update-pages-to-1-7-0.yml @@ -0,0 +1,5 @@ +--- +title: Update GitLab Pages to v1.7.0 +merge_request: +author: +type: other diff --git a/changelogs/unreleased/winh-notes-service-deleteNote.yml b/changelogs/unreleased/winh-notes-service-deleteNote.yml new file mode 100644 index 00000000000..cf2b37755a2 --- /dev/null +++ b/changelogs/unreleased/winh-notes-service-deleteNote.yml @@ -0,0 +1,5 @@ +--- +title: Remove deleteNote from notes service +merge_request: 30537 +author: Frank van Rest +type: other diff --git a/changelogs/unreleased/z-index-tools.yml b/changelogs/unreleased/z-index-tools.yml new file mode 100644 index 00000000000..1102612670b --- /dev/null +++ b/changelogs/unreleased/z-index-tools.yml @@ -0,0 +1,5 @@ +--- +title: 'Review Tools: Add large z-index to toolbar' +merge_request: 30583 +author: +type: fixed diff --git a/config/application.rb b/config/application.rb index edf8b3e87f9..de386506233 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,8 +1,14 @@ -require File.expand_path('boot', __dir__) +require_relative 'boot' -require 'rails/all' +# Based on https://github.com/rails/rails/blob/v5.2.3/railties/lib/rails/all.rb +# Only load the railties we need instead of loading everything +require 'active_record/railtie' +require 'action_controller/railtie' +require 'action_view/railtie' +require 'action_mailer/railtie' +require 'rails/test_unit/railtie' -Bundler.require(:default, Rails.env) +Bundler.require(*Rails.groups) module Gitlab class Application < Rails::Application @@ -25,6 +31,8 @@ module Gitlab # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. + config.active_record.sqlite3.represent_boolean_as_integer = true + # Sidekiq uses eager loading, but directories not in the standard Rails # directories must be added to the eager load paths: # https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code @@ -86,13 +94,6 @@ module Gitlab # Configure the default encoding used in templates for Ruby 1.9. config.encoding = "utf-8" - # ActionCable mount point. - # The default Rails' mount point is `/cable` which may conflict with existing - # namespaces/users. - # https://github.com/rails/rails/blob/5-0-stable/actioncable/lib/action_cable.rb#L38 - # Please change this value when configuring ActionCable for real usage. - config.action_cable.mount_path = "/-/cable" - # Configure sensitive parameters which will be filtered from the log file. # # Parameters filtered: @@ -272,5 +273,10 @@ module Gitlab Gitlab::Routing.add_helpers(project_url_helpers) Gitlab::Routing.add_helpers(MilestonesRoutingHelper) end + + # This makes generated cookies to be compatible with Rails 5.1 and older + # We can remove this when we're confident that there are no issues with the Rails 5.2 upgrade + # and we won't need to rollback to older versions + config.action_dispatch.use_authenticated_cookie_encryption = false end end diff --git a/config/environments/development.rb b/config/environments/development.rb index ac9b02b08d5..3881f1be152 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -7,6 +7,7 @@ Rails.application.configure do config.cache_classes = false # Show full error reports and disable caching + config.active_record.verbose_query_logs = true config.consider_all_requests_local = true config.action_controller.perform_caching = false diff --git a/config/environments/test.rb b/config/environments/test.rb index e7166882eea..153d16e4e55 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -23,6 +23,7 @@ Rails.application.configure do config.public_file_server.headers = { 'Cache-Control' => 'public, max-age=3600' } # Show full error reports and disable caching + config.active_record.verbose_query_logs = true config.consider_all_requests_local = true config.action_controller.perform_caching = false @@ -40,7 +41,7 @@ Rails.application.configure do # Print deprecation notices to the stderr config.active_support.deprecation = :stderr - config.eager_load = false + config.eager_load = true config.cache_store = :null_store diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 0b8a6607250..3a121addc98 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -55,7 +55,7 @@ if Settings.ldap['enabled'] || Rails.env.test? server['tls_options'] ||= {} if server['ssl_version'] || server['ca_file'] - Rails.logger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`' + Rails.logger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`' # rubocop:disable Gitlab/RailsLogger end if server['ssl_version'] diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 741c8ef1ca0..53c3eac3c74 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -17,7 +17,7 @@ def prometheus_default_multiproc_dir end Prometheus::Client.configure do |config| - config.logger = Rails.logger + config.logger = Rails.logger # rubocop:disable Gitlab/RailsLogger config.initial_mmap_file_size = 4 * 1024 diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb index e95157bfde5..151bce4d130 100644 --- a/config/initializers/active_record_data_types.rb +++ b/config/initializers/active_record_data_types.rb @@ -22,7 +22,7 @@ if Gitlab::Database.postgresql? # # When schema dumping, `timestamptz` columns will be output as # `t.datetime_with_timezone`. - def initialize_type_map(mapping) + def initialize_type_map(mapping = type_map) super mapping mapping.register_type 'timestamptz' do |_, _, sql_type| @@ -51,7 +51,7 @@ elsif Gitlab::Database.mysql? # # When schema dumping, `timestamp` columns will be output as # `t.datetime_with_timezone`. - def initialize_type_map(mapping) + def initialize_type_map(mapping = type_map) super mapping mapping.register_type(/timestamp/i) do |sql_type| diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb index 7fa37121efc..61f1d299960 100644 --- a/config/initializers/active_record_lifecycle.rb +++ b/config/initializers/active_record_lifecycle.rb @@ -7,7 +7,7 @@ if defined?(ActiveRecord::Base) && !Sidekiq.server? ActiveSupport.on_load(:active_record) do ActiveRecord::Base.establish_connection - Rails.logger.debug("ActiveRecord connection established") + Rails.logger.debug("ActiveRecord connection established") # rubocop:disable Gitlab/RailsLogger end end end @@ -18,6 +18,6 @@ if defined?(ActiveRecord::Base) # as there's no need for the master process to hold a connection ActiveRecord::Base.connection.disconnect! - Rails.logger.debug("ActiveRecord connection disconnected") + Rails.logger.debug("ActiveRecord connection disconnected") # rubocop:disable Gitlab/RailsLogger end end diff --git a/config/initializers/active_record_preloader.rb b/config/initializers/active_record_preloader.rb index 3b16014f302..a293909149e 100644 --- a/config/initializers/active_record_preloader.rb +++ b/config/initializers/active_record_preloader.rb @@ -1,9 +1,22 @@ module ActiveRecord module Associations class Preloader + class NullPreloader + def self.new(klass, owners, reflection, preload_scope) + self + end + + def self.run(preloader) + end + + def self.preloaded_records + [] + end + end + module NoCommitPreloader - def preloader_for(reflection, owners, rhs_klass) - return NullPreloader if rhs_klass == ::Commit + def preloader_for(reflection, owners) + return NullPreloader if owners.first.association(reflection.name).klass == ::Commit super end diff --git a/config/initializers/active_record_query_cache.rb b/config/initializers/active_record_query_cache.rb new file mode 100644 index 00000000000..61505a1edd3 --- /dev/null +++ b/config/initializers/active_record_query_cache.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +ActiveRecord::ConnectionAdapters::ConnectionPool.prepend Gitlab::Patch::ActiveRecordQueryCache diff --git a/config/initializers/active_record_verbose_query_logs.rb b/config/initializers/active_record_verbose_query_logs.rb deleted file mode 100644 index 1c5fbc8e830..00000000000 --- a/config/initializers/active_record_verbose_query_logs.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -# This is backport of https://github.com/rails/rails/pull/26815/files -# Enabled by default for every non-production environment - -module ActiveRecord - class LogSubscriber - module VerboseQueryLogs - def debug(progname = nil, &block) - return unless super - - log_query_source - end - - def log_query_source - source_line, line_number = extract_callstack(caller_locations) - - if source_line - if defined?(::Rails.root) - app_root = "#{::Rails.root}/".freeze - source_line = source_line.sub(app_root, "") - end - - logger.debug(" ↳ #{source_line}:#{line_number}") - end - end - - def extract_callstack(callstack) - line = callstack.find do |frame| - frame.absolute_path && !ignored_callstack(frame.absolute_path) - end - - offending_line = line || callstack.first - [ - offending_line.path, - offending_line.lineno, - offending_line.label - ] - end - - LOG_SUBSCRIBER_FILE = ActiveRecord::LogSubscriber.method(:logger).source_location.first - RAILS_GEM_ROOT = File.expand_path("../../../..", LOG_SUBSCRIBER_FILE) + "/" - APP_CONFIG_ROOT = File.expand_path("..", __dir__) + "/" - - def ignored_callstack(path) - path.start_with?(APP_CONFIG_ROOT, RAILS_GEM_ROOT, RbConfig::CONFIG["rubylibdir"]) - end - end - - if Rails.version.start_with?("5.2") - raise "Remove this monkey patch: #{__FILE__}" - else - prepend(VerboseQueryLogs) unless Rails.env.production? - end - end -end diff --git a/config/initializers/ar_speed_up_migration_checking.rb b/config/initializers/ar_speed_up_migration_checking.rb index f98b246db0b..c4ffcc54cb2 100644 --- a/config/initializers/ar_speed_up_migration_checking.rb +++ b/config/initializers/ar_speed_up_migration_checking.rb @@ -2,17 +2,14 @@ if Rails.env.test? require 'active_record/migration' module ActiveRecord - class Migrator - class << self - alias_method :migrations_unmemoized, :migrations + class MigrationContext + alias_method :migrations_unmemoized, :migrations - # This method is called a large number of times per rspec example, and - # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5 - # seconds per spec. - def migrations(paths) - @migrations ||= {} - (@migrations[paths] ||= migrations_unmemoized(paths)).dup - end + # This method is called a large number of times per rspec example, and + # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5 + # seconds per spec. + def migrations + @migrations ||= migrations_unmemoized end end end diff --git a/config/initializers/config_initializers_active_record_locking.rb b/config/initializers/config_initializers_active_record_locking.rb index 608d63223a3..915247826e9 100644 --- a/config/initializers/config_initializers_active_record_locking.rb +++ b/config/initializers/config_initializers_active_record_locking.rb @@ -22,10 +22,11 @@ module ActiveRecord # Patched because when `lock_version` is read as `0`, it may actually be `NULL` in the DB. possible_previous_lock_value = previous_lock_value.to_i == 0 ? [nil, 0] : previous_lock_value - affected_rows = self.class.unscoped._update_record( - arel_attributes_with_values(attribute_names), - self.class.primary_key => id_in_database, - locking_column => possible_previous_lock_value + affected_rows = self.class.unscoped.where( + locking_column => possible_previous_lock_value, + self.class.primary_key => id_in_database + ).update_all( + attributes_with_values_for_update(attribute_names) ) if affected_rows != 1 diff --git a/config/initializers/deprecations.rb b/config/initializers/deprecations.rb index 14616e726d9..0d096e34eb7 100644 --- a/config/initializers/deprecations.rb +++ b/config/initializers/deprecations.rb @@ -2,7 +2,7 @@ if Rails.env.development? || ENV['GITLAB_LEGACY_PATH_LOG_MESSAGE'] deprecator = ActiveSupport::Deprecation.new('11.0', 'GitLab') deprecator.behavior = -> (message, callstack) { - Rails.logger.warn("#{message}: #{callstack[1..20].join}") + Rails.logger.warn("#{message}: #{callstack[1..20].join}") # rubocop:disable Gitlab/RailsLogger } ActiveSupport::Deprecation.deprecate_methods(Gitlab::GitalyClient::StorageSettings, :legacy_disk_path, deprecator: deprecator) diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index 1ad93e14f7e..fbec28186eb 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -18,7 +18,7 @@ unless Sidekiq.server? .map { |k, v| { key: k, value: v } } payload = { - time: event.time.utc.iso8601(3), + time: Time.now.utc.iso8601(3), params: params, remote_ip: event.payload[:remote_ip], user_id: event.payload[:user_id], diff --git a/config/initializers/mysql_ignore_postgresql_options.rb b/config/initializers/mysql_ignore_postgresql_options.rb index 9a569be7674..e6a7d9bef52 100644 --- a/config/initializers/mysql_ignore_postgresql_options.rb +++ b/config/initializers/mysql_ignore_postgresql_options.rb @@ -15,7 +15,6 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) module ConnectionAdapters class Mysql2Adapter < AbstractMysqlAdapter alias_method :__gitlab_add_index, :add_index - alias_method :__gitlab_add_index_sql, :add_index_sql alias_method :__gitlab_add_index_options, :add_index_options def add_index(table_name, column_name, options = {}) @@ -24,12 +23,6 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) end end - def add_index_sql(table_name, column_name, options = {}) - unless options[:opclasses] - __gitlab_add_index_sql(table_name, column_name, options) - end - end - def add_index_options(table_name, column_name, options = {}) if options[:using] && options[:using] == :gin options = options.dup diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb index 56689bc8e74..68d53c4edbf 100644 --- a/config/initializers/postgresql_cte.rb +++ b/config/initializers/postgresql_cte.rb @@ -94,8 +94,8 @@ module ActiveRecord end end - def build_arel - arel = super() + def build_arel(aliases) + arel = super build_with(arel) if @values[:with] diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index 58f46b55725..246cf3482a4 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -14,8 +14,8 @@ if defined?(::Puma) && !Rails.env.test? Gitlab::Application.configure do |config| config.middleware.insert_before(Rack::Runtime, Rack::Timeout, - service_timeout: 60, - wait_timeout: 90) + service_timeout: ENV.fetch('GITLAB_RAILS_RACK_TIMEOUT', 60).to_i, + wait_timeout: ENV.fetch('GITLAB_RAILS_WAIT_TIMEOUT', 90).to_i) end observer = Gitlab::Cluster::RackTimeoutObserver.new diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index f9ef5d66bfa..166fb9b6916 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -72,7 +72,7 @@ Sidekiq.configure_server do |config| cron_jobs[k]['class'] = cron_jobs[k].delete('job_class') else cron_jobs.delete(k) - Rails.logger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") + Rails.logger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") # rubocop:disable Gitlab/RailsLogger end end Sidekiq::Cron::Job.load_from_hash! cron_jobs @@ -83,7 +83,7 @@ Sidekiq.configure_server do |config| Rails.application.config.database_configuration[Rails.env] db_config['pool'] = Sidekiq.options[:concurrency] ActiveRecord::Base.establish_connection(db_config) - Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}") + Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}") # rubocop:disable Gitlab/RailsLogger Gitlab.ee do Gitlab::Mirror.configure_cron_job! @@ -94,7 +94,7 @@ Sidekiq.configure_server do |config| Rails.configuration.geo_database['pool'] = Sidekiq.options[:concurrency] Geo::TrackingBase.establish_connection(Rails.configuration.geo_database) - Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{Geo::TrackingBase.connection_pool.size} (Geo tracking database)") + Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{Geo::TrackingBase.connection_pool.size} (Geo tracking database)") # rubocop:disable Gitlab/RailsLogger end end diff --git a/config/routes.rb b/config/routes.rb index a42fc037227..d208df62b4a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -82,7 +82,11 @@ Rails.application.routes.draw do resources :issues, only: [:index, :create, :update] end - resources :issues, module: :boards, only: [:index, :update] + resources :issues, module: :boards, only: [:index, :update] do + collection do + put :bulk_move, format: :json + end + end Gitlab.ee do resources :users, module: :boards, only: [:index] @@ -106,6 +110,22 @@ Rails.application.routes.draw do draw :jira_connect end + Gitlab.ee do + constraints(::Constraints::FeatureConstrainer.new(:analytics)) do + draw :analytics + end + end + + if ENV['GITLAB_CHAOS_SECRET'] || Rails.env.development? + resource :chaos, only: [] do + get :leakmem + get :cpu_spin + get :db_spin + get :sleep + get :kill + end + end + if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS'] get '/chaos/leakmem' => 'chaos#leakmem' get '/chaos/cpuspin' => 'chaos#cpuspin' diff --git a/config/routes/api.rb b/config/routes/api.rb index c6c17b5708e..d55bbdf6776 100644 --- a/config/routes/api.rb +++ b/config/routes/api.rb @@ -1,5 +1,5 @@ post '/api/graphql', to: 'graphql#execute' mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: '/api/graphql' -::API::API.logger Rails.logger +::API::API.logger Rails.logger # rubocop:disable Gitlab/RailsLogger mount ::API::API => '/' diff --git a/config/webpack.config.js b/config/webpack.config.js index a81590e8b8e..cd793743eb7 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -344,6 +344,8 @@ module.exports = { devtool: NO_SOURCEMAPS ? false : devtool, - // sqljs requires fs - node: { fs: 'empty' }, + node: { + fs: 'empty', // sqljs requires fs + setImmediate: false, + }, }; diff --git a/db/post_migrate/20180223124427_build_user_interacted_projects_table.rb b/db/post_migrate/20180223124427_build_user_interacted_projects_table.rb index fa332fd5c70..325895a5ddb 100644 --- a/db/post_migrate/20180223124427_build_user_interacted_projects_table.rb +++ b/db/post_migrate/20180223124427_build_user_interacted_projects_table.rb @@ -82,7 +82,7 @@ class BuildUserInteractedProjectsTable < ActiveRecord::Migration[4.2] iteration = 0 records = 0 begin - Rails.logger.info "Building user_interacted_projects table, batch ##{iteration}" + Rails.logger.info "Building user_interacted_projects table, batch ##{iteration}" # rubocop:disable Gitlab/RailsLogger result = execute <<~SQL INSERT INTO user_interacted_projects (user_id, project_id) SELECT e.user_id, e.project_id @@ -93,7 +93,7 @@ class BuildUserInteractedProjectsTable < ActiveRecord::Migration[4.2] SQL iteration += 1 records += result.cmd_tuples - Rails.logger.info "Building user_interacted_projects table, batch ##{iteration} complete, created #{records} overall" + Rails.logger.info "Building user_interacted_projects table, batch ##{iteration} complete, created #{records} overall" # rubocop:disable Gitlab/RailsLogger Kernel.sleep(SLEEP_TIME) if result.cmd_tuples > 0 end while result.cmd_tuples > 0 end diff --git a/db/post_migrate/20190628191740_schedule_fixing_names_of_user_namespaces.rb b/db/post_migrate/20190628191740_schedule_fixing_names_of_user_namespaces.rb new file mode 100644 index 00000000000..8fa7068b957 --- /dev/null +++ b/db/post_migrate/20190628191740_schedule_fixing_names_of_user_namespaces.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ScheduleFixingNamesOfUserNamespaces < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :user_namespaces, -> { where(type: nil) } + end + + class Route < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'routes' + + scope :project_routes, -> { where(source_type: 'Project') } + end + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + ScheduleFixingNamesOfUserNamespaces::Namespace.user_namespaces, + 'FixUserNamespaceNames', + 60.seconds, + batch_size: 5000 + ) + + queue_background_migration_jobs_by_range_at_intervals( + ScheduleFixingNamesOfUserNamespaces::Route.project_routes, + 'FixUserProjectRouteNames', + 60.seconds, + batch_size: 5000 + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20190702173936_populate_remaining_merge_request_assignees.rb b/db/post_migrate/20190702173936_populate_remaining_merge_request_assignees.rb new file mode 100644 index 00000000000..c435b94015d --- /dev/null +++ b/db/post_migrate/20190702173936_populate_remaining_merge_request_assignees.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PopulateRemainingMergeRequestAssignees < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 10_000 + MIGRATION = 'PopulateMergeRequestAssigneesTable' + + disable_ddl_transaction! + + def up + Gitlab::BackgroundMigration.steal(MIGRATION) + + Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable.new.perform_all_sync(batch_size: BATCH_SIZE) + end +end diff --git a/db/schema.rb b/db/schema.rb index 9a8b64689bd..9d28c5d2383 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,11 +10,11 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190703130053) do +ActiveRecord::Schema.define(version: 2019_07_03_130053) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" enable_extension "pg_trgm" + enable_extension "plpgsql" create_table "abuse_reports", id: :serial, force: :cascade do |t| t.integer "reporter_id" diff --git a/doc/administration/auth/authentiq.md b/doc/administration/auth/authentiq.md index 726622d8599..835c97c0288 100644 --- a/doc/administration/auth/authentiq.md +++ b/doc/administration/auth/authentiq.md @@ -8,47 +8,48 @@ Authentiq will generate a Client ID and the accompanying Client Secret for you t 1. On your GitLab server, open the configuration file: - For omnibus installation - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + For omnibus installation - For installations from source: + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - ```sh - sudo -u git -H editor /home/git/gitlab/config/gitlab.yml - ``` + For installations from source: + + ```sh + sudo -u git -H editor /home/git/gitlab/config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings to enable single sign-on and add Authentiq as an OAuth provider. 1. Add the provider configuration for Authentiq: - For Omnibus packages: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "authentiq", - "app_id" => "YOUR_CLIENT_ID", - "app_secret" => "YOUR_CLIENT_SECRET", - "args" => { - "scope": 'aq:name email~rs address aq:push' - } - } - ] - ``` - - For installations from source: - - ```yaml - - { name: 'authentiq', - app_id: 'YOUR_CLIENT_ID', - app_secret: 'YOUR_CLIENT_SECRET', - args: { - scope: 'aq:name email~rs address aq:push' - } - } - ``` + For Omnibus packages: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "authentiq", + "app_id" => "YOUR_CLIENT_ID", + "app_secret" => "YOUR_CLIENT_SECRET", + "args" => { + "scope": 'aq:name email~rs address aq:push' + } + } + ] + ``` + + For installations from source: + + ```yaml + - { name: 'authentiq', + app_id: 'YOUR_CLIENT_ID', + app_secret: 'YOUR_CLIENT_SECRET', + args: { + scope: 'aq:name email~rs address aq:push' + } + } + ``` 1. The `scope` is set to request the user's name, email (required and signed), and permission to send push notifications to sign in on subsequent visits. See [OmniAuth Authentiq strategy](https://github.com/AuthentiqID/omniauth-authentiq/wiki/Scopes,-callback-url-configuration-and-responses) for more information on scopes and modifiers. diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 6db74958d5a..86c7bad2ebf 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -6,55 +6,56 @@ 1. Go through the 'Add application' steps, entering the appropriate details. The screenshot below shows an example configuration. -  +  ## Configure GitLab 1. On your GitLab server, open the configuration file. - **Omnibus:** + **Omnibus:** - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - **Source:** + **Source:** - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - **Omnibus:** - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "crowd", - "args" => { - "crowd_server_url" => "CROWD_SERVER_URL", - "application_name" => "YOUR_APP_NAME", - "application_password" => "YOUR_APP_PASSWORD" - } - } - ] - ``` - - **Source:** - - ``` - - { name: 'crowd', - args: { - crowd_server_url: 'CROWD_SERVER_URL', - application_name: 'YOUR_APP_NAME', - application_password: 'YOUR_APP_PASSWORD' } } - ``` + **Omnibus:** + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "crowd", + "args" => { + "crowd_server_url" => "CROWD_SERVER_URL", + "application_name" => "YOUR_APP_NAME", + "application_password" => "YOUR_APP_PASSWORD" + } + } + ] + ``` + + **Source:** + + ``` + - { name: 'crowd', + args: { + crowd_server_url: 'CROWD_SERVER_URL', + application_name: 'YOUR_APP_NAME', + application_password: 'YOUR_APP_PASSWORD' } } + ``` + 1. Change `CROWD_SERVER_URL` to the URL of your Crowd server. 1. Change `YOUR_APP_NAME` to the application name from Crowd applications page. 1. Change `YOUR_APP_PASSWORD` to the application password you've set. @@ -77,4 +78,4 @@ could not authorize you from Crowd because invalid credentials Please make sure the Crowd users who need to login to GitLab are authorized to [the application](#configure-a-new-crowd-application) in the step of **Authorisation**. This could be verified by try "Authentication test" for Crowd as of 2.11. -
\ No newline at end of file + diff --git a/doc/administration/auth/google_secure_ldap.md b/doc/administration/auth/google_secure_ldap.md index 1db5bb4bc3f..0e6d7ff1df1 100644 --- a/doc/administration/auth/google_secure_ldap.md +++ b/doc/administration/auth/google_secure_ldap.md @@ -66,7 +66,7 @@ values obtained during the LDAP client configuration earlier: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby + ```ruby gitlab_rails['ldap_enabled'] = true gitlab_rails['ldap_servers'] = YAML.load <<-EOS # remember to close this block with 'EOS' below main: # 'main' is the GitLab 'provider ID' of this LDAP server @@ -127,7 +127,7 @@ values obtained during the LDAP client configuration earlier: AcZSFJQjdg5BTyvdEDhaYUKGdRw= -----END PRIVATE KEY----- EOS - ``` + ``` 1. Save the file and [reconfigure] GitLab for the changes to take effect. @@ -137,7 +137,7 @@ values obtained during the LDAP client configuration earlier: 1. Edit `config/gitlab.yml`: - ```yaml + ```yaml ldap: enabled: true servers: diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 497298503ad..7db22bdd5df 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -3,65 +3,65 @@ To enable the JWT OmniAuth provider, you must register your application with JWT. JWT will provide you with a secret key for you to use. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For Omnibus GitLab: + For Omnibus GitLab: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration. +1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. +1. Add the provider configuration. - For Omnibus GitLab: + For Omnibus GitLab: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { name: 'jwt', - args: { - secret: 'YOUR_APP_SECRET', - algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' - uid_claim: 'email', - required_claims: ['name', 'email'], - info_maps: { name: 'name', email: 'email' }, - auth_url: 'https://example.com/', - valid_within: 3600 # 1 hour - } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { name: 'jwt', + args: { + secret: 'YOUR_APP_SECRET', + algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' + uid_claim: 'email', + required_claims: ['name', 'email'], + info_maps: { name: 'name', email: 'email' }, + auth_url: 'https://example.com/', + valid_within: 3600 # 1 hour + } + } + ] + ``` - For installation from source: + For installation from source: - ``` - - { name: 'jwt', - args: { - secret: 'YOUR_APP_SECRET', - algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' - uid_claim: 'email', - required_claims: ['name', 'email'], - info_map: { name: 'name', email: 'email' }, - auth_url: 'https://example.com/', - valid_within: 3600 # 1 hour - } - } - ``` + ``` + - { name: 'jwt', + args: { + secret: 'YOUR_APP_SECRET', + algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' + uid_claim: 'email', + required_claims: ['name', 'email'], + info_map: { name: 'name', email: 'email' }, + auth_url: 'https://example.com/', + valid_within: 3600 # 1 hour + } + } + ``` - NOTE: **Note:** For more information on each configuration option refer to - the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). + NOTE: **Note:** For more information on each configuration option refer to + the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). -1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. -1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. +1. Save the configuration file. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a JWT icon below the regular sign in form. Click the icon to begin the authentication process. JWT will ask the user to diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 1a8af0827ee..2afac23c20c 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -85,19 +85,19 @@ following. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## Base where we can search for groups + ## + ## Ex. ou=groups,dc=gitlab,dc=example + ## + ## + group_base: ou=groups,dc=example,dc=com + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -105,14 +105,14 @@ following. 1. Edit `/home/git/gitlab/config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -140,30 +140,30 @@ group, as opposed to the full DN. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - - ## - ## The CN of a group containing GitLab administrators - ## - ## Ex. administrators - ## - ## Note: Not `cn=administrators` or the full DN - ## - ## - admin_group: my_admin_group - - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## Base where we can search for groups + ## + ## Ex. ou=groups,dc=gitlab,dc=example + ## + ## + group_base: ou=groups,dc=example,dc=com + + ## + ## The CN of a group containing GitLab administrators + ## + ## Ex. administrators + ## + ## Note: Not `cn=administrators` or the full DN + ## + ## + admin_group: my_admin_group + + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -171,15 +171,15 @@ group, as opposed to the full DN. 1. Edit `/home/git/gitlab/config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - admin_group: my_admin_group - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + admin_group: my_admin_group + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -191,7 +191,6 @@ to lock down user abilities to invite new members to a group. When enabled follo 1. Only administrator can manage memberships of any group including access levels. 2. Users are not allowed to share project with other groups or invite members to a project created in a group. - ## Adjusting LDAP user sync schedule > Introduced in GitLab Enterprise Edition Starter. @@ -211,9 +210,9 @@ sync to run once every 12 hours at the top of the hour. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" - ``` + ```ruby + gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -221,11 +220,11 @@ sync to run once every 12 hours at the top of the hour. 1. Edit `config/gitlab.yaml`: - ```yaml - cron_jobs: - ldap_sync_worker_cron: - "0 */12 * * *" - ``` + ```yaml + cron_jobs: + ldap_sync_worker_cron: + "0 */12 * * *" + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -252,9 +251,9 @@ sync to run once every 2 hours at the top of the hour. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" - ``` + ```ruby + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -262,11 +261,11 @@ sync to run once every 2 hours at the top of the hour. 1. Edit `config/gitlab.yaml`: - ```yaml - cron_jobs: - ldap_group_sync_worker_cron: - "*/30 * * * *" - ``` + ```yaml + cron_jobs: + ldap_group_sync_worker_cron: + "*/30 * * * *" + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -283,20 +282,20 @@ task. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## An array of CNs of groups containing users that should be considered external - ## - ## Ex. ['interns', 'contractors'] - ## - ## Note: Not `cn=interns` or the full DN - ## - external_groups: ['interns', 'contractors'] - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## An array of CNs of groups containing users that should be considered external + ## + ## Ex. ['interns', 'contractors'] + ## + ## Note: Not `cn=interns` or the full DN + ## + external_groups: ['interns', 'contractors'] + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -304,14 +303,14 @@ task. 1. Edit `config/gitlab.yaml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - external_groups: ['interns', 'contractors'] - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + external_groups: ['interns', 'contractors'] + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -436,66 +435,71 @@ step of the sync. 1. Start a Rails console - ```bash - # For Omnibus installations - sudo gitlab-rails console + ```bash + # For Omnibus installations + sudo gitlab-rails console - # For installations from source - sudo -u git -H bundle exec rails console production - ``` + # For installations from source + sudo -u git -H bundle exec rails console production + ``` 1. Set the log level to debug (only for this session): - ```ruby - Rails.logger.level = Logger::DEBUG - ``` + ```ruby + Rails.logger.level = Logger::DEBUG + ``` + 1. Choose a GitLab group to test with. This group should have an LDAP group link already configured. If the output is `nil`, the group could not be found. If a bunch of group attributes are output, your group was found successfully. - ```ruby - group = Group.find_by(name: 'my_group') + ```ruby + group = Group.find_by(name: 'my_group') + + # Output + => #<Group:0x007fe825196558 id: 1234, name: "my_group"...> + ``` - # Output - => #<Group:0x007fe825196558 id: 1234, name: "my_group"...> - ``` 1. Run a group sync for this particular group. - ```ruby - EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) - ``` + ```ruby + EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) + ``` + 1. Look through the output of the sync. See [example log output](#example-log-output) below for more information about the output. 1. If you still aren't able to see why the user isn't being added, query the LDAP group directly to see what members are listed. Still in the Rails console, run the following query: - ```ruby - adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider - ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) + ```ruby + adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider + ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) + + # Output + => #<EE::Gitlab::Auth::LDAP::Group:0x007fcbdd0bb6d8 + ``` - # Output - => #<EE::Gitlab::Auth::LDAP::Group:0x007fcbdd0bb6d8 - ``` 1. Query the LDAP group's member DNs and see if the user's DN is in the list. One of the DNs here should match the 'Identifier' from the LDAP identity checked earlier. If it doesn't, the user does not appear to be in the LDAP group. - ```ruby - ldap_group.member_dns + ```ruby + ldap_group.member_dns + + # Output + => ["uid=john,ou=people,dc=example,dc=com", "uid=mary,ou=people,dc=example,dc=com"] + ``` - # Output - => ["uid=john,ou=people,dc=example,dc=com", "uid=mary,ou=people,dc=example,dc=com"] - ``` 1. Some LDAP servers don't store members by DN. Rather, they use UIDs instead. If you didn't see results from the last query, try querying by UIDs instead. - ```ruby - ldap_group.member_uids + ```ruby + ldap_group.member_uids - # Output - => ['john','mary'] - ``` + # Output + => ['john','mary'] + ``` #### Example log output diff --git a/doc/administration/auth/ldap.md b/doc/administration/auth/ldap.md index 2144f5753a8..86e6be5f4fa 100644 --- a/doc/administration/auth/ldap.md +++ b/doc/administration/auth/ldap.md @@ -398,30 +398,30 @@ The `user_filter` DN can contain special characters. For example: - A comma: - ``` - OU=GitLab, Inc,DC=gitlab,DC=com - ``` + ``` + OU=GitLab, Inc,DC=gitlab,DC=com + ``` - Open and close brackets: - ``` - OU=Gitlab (Inc),DC=gitlab,DC=com - ``` + ``` + OU=Gitlab (Inc),DC=gitlab,DC=com + ``` - These characters must be escaped as documented in - [RFC 4515](https://tools.ietf.org/search/rfc4515). + These characters must be escaped as documented in + [RFC 4515](https://tools.ietf.org/search/rfc4515). - Escape commas with `\2C`. For example: - ``` - OU=GitLab\2C Inc,DC=gitlab,DC=com - ``` + ``` + OU=GitLab\2C Inc,DC=gitlab,DC=com + ``` - Escape open and close brackets with `\28` and `\29`, respectively. For example: - ``` - OU=Gitlab \28Inc\29,DC=gitlab,DC=com - ``` + ``` + OU=Gitlab \28Inc\29,DC=gitlab,DC=com + ``` ## Enabling LDAP sign-in for existing GitLab users @@ -445,13 +445,13 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - # snip... - lowercase_usernames: true - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + # snip... + lowercase_usernames: true + EOS + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -459,14 +459,14 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. Edit `config/gitlab.yaml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - lowercase_usernames: true - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + lowercase_usernames: true + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -519,13 +519,13 @@ ldapsearch -H ldaps://$host:$port -D "$bind_dn" -y bind_dn_password.txt -b "$ba - Run the following check command to make sure that the LDAP settings are correct and GitLab can see your users: - ```bash - # For Omnibus installations - sudo gitlab-rake gitlab:ldap:check + ```bash + # For Omnibus installations + sudo gitlab-rake gitlab:ldap:check - # For installations from source - sudo -u git -H bundle exec rake gitlab:ldap:check RAILS_ENV=production - ``` + # For installations from source + sudo -u git -H bundle exec rake gitlab:ldap:check RAILS_ENV=production + ``` ### Connection Refused diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 6e48add6930..454da8c2866 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -5,76 +5,76 @@ GitLab can use [OpenID Connect](https://openid.net/specs/openid-connect-core-1_0 To enable the OpenID Connect OmniAuth provider, you must register your application with an OpenID Connect provider. The OpenID Connect will provide you with a client details and secret for you to use. -1. On your GitLab server, open the configuration file. - - For Omnibus GitLab: - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - - See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. - -1. Add the provider configuration. - - For Omnibus GitLab: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { 'name' => 'openid_connect', - 'label' => '<your_oidc_label>', - 'args' => { - "name' => 'openid_connect', - 'scope' => ['openid','profile'], - 'response_type' => 'code', - 'issuer' => '<your_oidc_url>', - 'discovery' => true, - 'client_auth_method' => 'query', - 'uid_field' => '<uid_field>', - 'client_options' => { - 'identifier' => '<your_oidc_client_id>', - 'secret' => '<your_oidc_client_secret>', - 'redirect_uri' => '<your_gitlab_url>/users/auth/openid_connect/callback' - } - } - } - ] - ``` - - For installation from source: - - ```yaml - - { name: 'openid_connect', - label: '<your_oidc_label>', - args: { - name: 'openid_connect', - scope: ['openid','profile'], - response_type: 'code', - issuer: '<your_oidc_url>', - discovery: true, - client_auth_method: 'query', - uid_field: '<uid_field>', - client_options: { - identifier: '<your_oidc_client_id>', - secret: '<your_oidc_client_secret>', - redirect_uri: '<your_gitlab_url>/users/auth/openid_connect/callback' - } - } - } - ``` - - > **Note:** - > - > - For more information on each configuration option refer to - the [OmniAuth OpenID Connect usage documentation](https://github.com/m0n9oose/omniauth_openid_connect#usage) and - the [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). +1. On your GitLab server, open the configuration file. + + For Omnibus GitLab: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + + See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. + +1. Add the provider configuration. + + For Omnibus GitLab: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { 'name' => 'openid_connect', + 'label' => '<your_oidc_label>', + 'args' => { + "name' => 'openid_connect', + 'scope' => ['openid','profile'], + 'response_type' => 'code', + 'issuer' => '<your_oidc_url>', + 'discovery' => true, + 'client_auth_method' => 'query', + 'uid_field' => '<uid_field>', + 'client_options' => { + 'identifier' => '<your_oidc_client_id>', + 'secret' => '<your_oidc_client_secret>', + 'redirect_uri' => '<your_gitlab_url>/users/auth/openid_connect/callback' + } + } + } + ] + ``` + + For installation from source: + + ```yaml + - { name: 'openid_connect', + label: '<your_oidc_label>', + args: { + name: 'openid_connect', + scope: ['openid','profile'], + response_type: 'code', + issuer: '<your_oidc_url>', + discovery: true, + client_auth_method: 'query', + uid_field: '<uid_field>', + client_options: { + identifier: '<your_oidc_client_id>', + secret: '<your_oidc_client_secret>', + redirect_uri: '<your_gitlab_url>/users/auth/openid_connect/callback' + } + } + } + ``` + + > **Note:** + > + > - For more information on each configuration option refer to + the [OmniAuth OpenID Connect usage documentation](https://github.com/m0n9oose/omniauth_openid_connect#usage) and + the [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). 1. For the configuration above, change the values for the provider to match your OpenID Connect client setup. Use the following as a guide: - `<your_oidc_label>` is the label that will be displayed on the login page. diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index aa4e1b0d2e0..566003ba708 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -16,7 +16,7 @@ The following documentation enables Okta as a SAML provider. 1. Next, you'll need the to fill in the SAML general config. Here's an example image. -  +  1. The last part of the configuration is the feedback section where you can just say you're a customer and creating an app for internal use. @@ -24,7 +24,7 @@ The following documentation enables Okta as a SAML provider. profile. Click on the SAML 2.0 config instructions button which should look like the following: -  +  1. On the screen that comes up take note of the **Identity Provider Single Sign-On URL** which you'll use for the @@ -38,112 +38,112 @@ Now that the Okta app is configured, it's time to enable it in GitLab. ## Configure GitLab -1. On your GitLab server, open the configuration file: - - **For Omnibus GitLab installations** - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - **For installations from source** - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - -1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) - for initial settings. - -1. To allow your users to use Okta to sign up without having to manually create - an account first, don't forget to add the following values to your - configuration: - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] - gitlab_rails['omniauth_block_auto_created_users'] = false - ``` - - **For installations from source** - - ```yaml - allow_single_sign_on: ["saml"] - block_auto_created_users: false - ``` - -1. You can also automatically link Okta users with existing GitLab users if - their email addresses match by adding the following setting: - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_auto_link_saml_user'] = true - ``` - - **For installations from source** - - ```yaml - auto_link_saml_user: true - ``` - -1. Add the provider configuration. - - >**Notes:** - > - >- Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint - of GitLab (append `users/auth/saml/callback` to the HTTPS URL of your GitLab - installation to generate the correct value). - > - >- To get the `idp_cert_fingerprint` fingerprint, first download the - certificate from the Okta app you registered and then run: - `openssl x509 -in okta.cert -noout -fingerprint`. Substitute `okta.cert` - with the location of your certificate. - > - >- Change the value of `idp_sso_target_url`, with the value of the - **Identity Provider Single Sign-On URL** from the step when you - configured the Okta app. - > - >- Change the value of `issuer` to the value of the **Audience Restriction** from your Okta app configuration. This will identify GitLab - to the IdP. - > - >- Leave `name_identifier_format` as-is. - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - }, - label: 'Okta' # optional label for SAML login button, defaults to "Saml" - } - ] - ``` - - **For installations from source** - - ```yaml - - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - }, - label: 'Okta' # optional label for SAML login button, defaults to "Saml" - } - ``` +1. On your GitLab server, open the configuration file: + + **For Omnibus GitLab installations** + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + **For installations from source** + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) + for initial settings. + +1. To allow your users to use Okta to sign up without having to manually create + an account first, don't forget to add the following values to your + configuration: + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] + gitlab_rails['omniauth_block_auto_created_users'] = false + ``` + + **For installations from source** + + ```yaml + allow_single_sign_on: ["saml"] + block_auto_created_users: false + ``` + +1. You can also automatically link Okta users with existing GitLab users if + their email addresses match by adding the following setting: + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_auto_link_saml_user'] = true + ``` + + **For installations from source** + + ```yaml + auto_link_saml_user: true + ``` + +1. Add the provider configuration. + + >**Notes:** + > + >- Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint + of GitLab (append `users/auth/saml/callback` to the HTTPS URL of your GitLab + installation to generate the correct value). + > + >- To get the `idp_cert_fingerprint` fingerprint, first download the + certificate from the Okta app you registered and then run: + `openssl x509 -in okta.cert -noout -fingerprint`. Substitute `okta.cert` + with the location of your certificate. + > + >- Change the value of `idp_sso_target_url`, with the value of the + **Identity Provider Single Sign-On URL** from the step when you + configured the Okta app. + > + >- Change the value of `issuer` to the value of the **Audience Restriction** from your Okta app configuration. This will identify GitLab + to the IdP. + > + >- Leave `name_identifier_format` as-is. + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + }, + label: 'Okta' # optional label for SAML login button, defaults to "Saml" + } + ] + ``` + + **For installations from source** + + ```yaml + - { + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + }, + label: 'Okta' # optional label for SAML login button, defaults to "Saml" + } + ``` 1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../restart_gitlab.md#installations-from-source) GitLab for Omnibus and installations from source respectively for the changes to take effect. diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index a0d4e9ef3b5..e47751e0cc5 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -56,11 +56,11 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['smartcard_enabled'] = true - gitlab_rails['smartcard_ca_file'] = "/etc/ssl/certs/CA.pem" - gitlab_rails['smartcard_client_certificate_required_port'] = 3444 - ``` + ```ruby + gitlab_rails['smartcard_enabled'] = true + gitlab_rails['smartcard_ca_file'] = "/etc/ssl/certs/CA.pem" + gitlab_rails['smartcard_client_certificate_required_port'] = 3444 + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -154,15 +154,15 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - # snip... - # Enable smartcard authentication against the LDAP server. Valid values - # are "false", "optional", and "required". - smartcard_auth: optional - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + # snip... + # Enable smartcard authentication against the LDAP server. Valid values + # are "false", "optional", and "required". + smartcard_auth: optional + EOS + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -171,16 +171,16 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - # Enable smartcard authentication against the LDAP server. Valid values - # are "false", "optional", and "required". - smartcard_auth: optional - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + # Enable smartcard authentication against the LDAP server. Valid values + # are "false", "optional", and "required". + smartcard_auth: optional + ``` 1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. @@ -191,9 +191,9 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['smartcard_required_for_git_access'] = true - ``` + ```ruby + gitlab_rails['smartcard_required_for_git_access'] = true + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -202,13 +202,13 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `config/gitlab.yml`: - ```yaml - ## Smartcard authentication settings - smartcard: - # snip... - # Browser session with smartcard sign-in is required for Git access - required_for_git_access: true - ``` + ```yaml + ## Smartcard authentication settings + smartcard: + # snip... + # Browser session with smartcard sign-in is required for Git access + required_for_git_access: true + ``` 1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index 04f52783d22..3e3054af509 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -696,12 +696,11 @@ project or branch name. Special characters can include: - Trailing hyphen/dash - Double hyphen/dash -To get around this, you can [change the group path](../user/group/index.md#changing-a-groups-path), -[change the project path](../user/project/settings/index.md#renaming-a-repository) or change the -branch name. Another option is to create a [push rule](../push_rules/push_rules.html) to prevent +To get around this, you can [change the group path](../user/group/index.md#changing-a-groups-path), +[change the project path](../user/project/settings/index.md#renaming-a-repository) or change the +branch name. Another option is to create a [push rule](../push_rules/push_rules.html) to prevent this at the instance level. - [ce-18239]: https://gitlab.com/gitlab-org/gitlab-ce/issues/18239 [docker-insecure-self-signed]: https://docs.docker.com/registry/insecure/#use-self-signed-certificates [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/dependency_proxy.md b/doc/administration/dependency_proxy.md index 776c60703fc..d2c52b67e67 100644 --- a/doc/administration/dependency_proxy.md +++ b/doc/administration/dependency_proxy.md @@ -70,6 +70,7 @@ To change the local storage path: enabled: true storage_path: shared/dependency_proxy ``` + 1. [Restart GitLab] for the changes to take effect. ### Using object storage diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index 30d35df25c7..64d7ef2d609 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -30,7 +30,7 @@ To bring the former **primary** node up to date: `sudo systemctl enable gitlab-runsvdir`. For CentOS 6, you need to install the GitLab instance from scratch and set it up as a **secondary** node by following [Setup instructions][setup-geo]. In this case, you don't need to follow the next step. - + NOTE: **Note:** If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record) for this node during disaster recovery procedure you may need to [block all the writes to this node](planned_failover.md#prevent-updates-to-the-primary-node) diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 393326c3347..0c2d48854f0 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -187,7 +187,7 @@ access to the **primary** node during the maintenance window. before it is completed will cause the work to be lost. 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the following conditions to be true of the **secondary** node you are failing over to: - + - All replication meters to each 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. - Database replication lag is 0ms. diff --git a/doc/administration/geo/replication/database.md b/doc/administration/geo/replication/database.md index 71a1ce87833..9272287a4a7 100644 --- a/doc/administration/geo/replication/database.md +++ b/doc/administration/geo/replication/database.md @@ -116,7 +116,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o by default. However, Geo requires the **secondary** node to be able to connect to the **primary** node's database. For this reason, we need the address of each node. - + NOTE: **Note:** For external PostgreSQL instances, see [additional instructions](external_database.md). If you are using a cloud provider, you can lookup the addresses for each @@ -424,22 +424,22 @@ data before running `pg_basebackup`. - If PostgreSQL is listening on a non-standard port, add `--port=` as well. - If your database is too large to be transferred in 30 minutes, you will need - to increase the timeout, e.g., `--backup-timeout=3600` if you expect the - initial replication to take under an hour. - - Pass `--sslmode=disable` to skip PostgreSQL TLS authentication altogether - (e.g., you know the network path is secure, or you are using a site-to-site - VPN). This is **not** safe over the public Internet! - - You can read more details about each `sslmode` in the - [PostgreSQL documentation][pg-docs-ssl]; - the instructions above are carefully written to ensure protection against - both passive eavesdroppers and active "man-in-the-middle" attackers. - - Change the `--slot-name` to the name of the replication slot - to be used on the **primary** database. The script will attempt to create the - replication slot automatically if it does not exist. - - If you're repurposing an old server into a Geo **secondary** node, you'll need to - add `--force` to the command line. - - When not in a production machine you can disable backup step if you - really sure this is what you want by adding `--skip-backup` + to increase the timeout, e.g., `--backup-timeout=3600` if you expect the + initial replication to take under an hour. + - Pass `--sslmode=disable` to skip PostgreSQL TLS authentication altogether + (e.g., you know the network path is secure, or you are using a site-to-site + VPN). This is **not** safe over the public Internet! + - You can read more details about each `sslmode` in the + [PostgreSQL documentation][pg-docs-ssl]; + the instructions above are carefully written to ensure protection against + both passive eavesdroppers and active "man-in-the-middle" attackers. + - Change the `--slot-name` to the name of the replication slot + to be used on the **primary** database. The script will attempt to create the + replication slot automatically if it does not exist. + - If you're repurposing an old server into a Geo **secondary** node, you'll need to + add `--force` to the command line. + - When not in a production machine you can disable backup step if you + really sure this is what you want by adding `--skip-backup` The replication process is now complete. diff --git a/doc/administration/geo/replication/remove_geo_node.md b/doc/administration/geo/replication/remove_geo_node.md index 4f64e21f8ef..e24eb2bd428 100644 --- a/doc/administration/geo/replication/remove_geo_node.md +++ b/doc/administration/geo/replication/remove_geo_node.md @@ -19,10 +19,10 @@ Once removed from the Geo admin page, you must stop and uninstall the **secondar ```bash # Stop gitlab and remove its supervision process sudo gitlab-ctl uninstall - + # Debian/Ubuntu sudo dpkg --remove gitlab-ee - + # Redhat/Centos sudo rpm --erase gitlab-ee ``` @@ -32,9 +32,9 @@ Once GitLab has been uninstalled from the **secondary** node, the replication sl 1. On the **primary** node, start a PostgreSQL console session: ```bash - sudo gitlab-psql + sudo gitlab-psql ``` - + NOTE: **Note:** Using `gitlab-rails dbconsole` will not work, because managing replication slots requires superuser permissions. @@ -43,9 +43,9 @@ Once GitLab has been uninstalled from the **secondary** node, the replication sl ```sql SELECT * FROM pg_replication_slots; ``` - + 1. Remove the replication slot for the **secondary** node: ```sql SELECT pg_drop_replication_slot('<name_of_slot>'); - ``` + ``` diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index e81d2741082..4317d14ba68 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -162,14 +162,14 @@ contention due to certain workloads. - **Status:** Work-in-progress - **Supported Users (approximate):** 10,000 -- **Related Issues:** [gitlab-com/support/support-team-meta#1513](https://gitlab.com/gitlab-com/support/support-team-meta/issues/1513), +- **Related Issues:** [gitlab-com/support/support-team-meta#1513](https://gitlab.com/gitlab-com/support/support-team-meta/issues/1513), [gitlab-org/quality/team-tasks#110](https://gitlab.com/gitlab-org/quality/team-tasks/issues/110) The Support and Quality teams are in the process of building and performance testing an environment that will support about 10,000 users. The specifications below -are a work-in-progress representation of the work so far. Quality will be -certifying this environment in FY20-Q2. The specifications may be adjusted -prior to certification based on performance testing. +are a work-in-progress representation of the work so far. Quality will be +certifying this environment in FY20-Q2. The specifications may be adjusted +prior to certification based on performance testing. - 3 PostgreSQL - 4 CPU, 8GB RAM per node - 1 PgBouncer - 2 CPU, 4GB RAM @@ -211,4 +211,3 @@ separately: 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) 1. [Monitoring node (Prometheus and Grafana)](monitoring_node.md) - diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index 49199b659bc..1f93c8130d3 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -26,27 +26,27 @@ On each Consul node perform the following: 1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - ```ruby - # Disable all components except Consul - roles ['consul_role'] - - # START user configuration - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - server: true, - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - - # Disable auto migrations - gitlab_rails['auto_migrate'] = false - # - # END user configuration - ``` - - > `consul_role` was introduced with GitLab 10.3 + ```ruby + # Disable all components except Consul + roles ['consul_role'] + + # START user configuration + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + server: true, + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + # + # END user configuration + ``` + + > `consul_role` was introduced with GitLab 10.3 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -77,6 +77,7 @@ check the [Troubleshooting section](#troubleshooting) before proceeding. ### Checking cluster membership To see which nodes are part of the cluster, run the following on any member in the cluster + ``` # /opt/gitlab/embedded/bin/consul members Node Address Status Type Build Protocol DC @@ -112,18 +113,18 @@ You will see messages like the following in `gitlab-ctl tail consul` output if y 2017-09-25_19:53:41.74356 2017/09/25 19:53:41 [ERR] agent: failed to sync remote state: No cluster leader ``` - To fix this: 1. Pick an address on each node that all of the other nodes can reach this node through. 1. Update your `/etc/gitlab/gitlab.rb` - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + 1. Run `gitlab-ctl reconfigure` If you still see the errors, you may have to [erase the consul database and reinitialize](#recreate-from-scratch) on the affected node. @@ -144,12 +145,13 @@ To fix this: 1. Pick an address on the node that all of the other nodes can reach this node through. 1. Update your `/etc/gitlab/gitlab.rb` - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + 1. Run `gitlab-ctl reconfigure` ### Outage recovery @@ -157,6 +159,7 @@ To fix this: If you lost enough server agents in the cluster to break quorum, then the cluster is considered failed, and it will not function without manual intervenetion. #### Recreate from scratch + By default, GitLab does not store anything in the consul cluster that cannot be recreated. To erase the consul database and reinitialize ``` @@ -168,4 +171,5 @@ By default, GitLab does not store anything in the consul cluster that cannot be After this, the cluster should start back up, and the server agents rejoin. Shortly after that, the client agents should rejoin as well. #### Recover a failed cluster + If you have taken advantage of consul to store other data, and want to restore the failed cluster, please follow the [Consul guide](https://www.consul.io/docs/guides/outage.html) to recover a failed cluster. diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index c32a73080ff..1702a731647 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -33,15 +33,15 @@ deploy the bundled PostgreSQL. 1. SSH into the PostgreSQL server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. + - Do not complete any other steps on the download page. 1. Generate a password hash for PostgreSQL. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next step as the value of `POSTGRESQL_PASSWORD_HASH`. - ```sh - sudo gitlab-ctl pg-password-md5 gitlab - ``` + ```sh + sudo gitlab-ctl pg-password-md5 gitlab + ``` 1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder values appropriately. @@ -51,32 +51,32 @@ deploy the bundled PostgreSQL. addresses of the GitLab application servers that will connect to the database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` - ```ruby - # Disable all components except PostgreSQL - roles ['postgres_role'] - repmgr['enable'] = false - consul['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - redis_exporter['enable'] = false - gitlab_monitor['enable'] = false - - postgresql['listen_address'] = '0.0.0.0' - postgresql['port'] = 5432 - - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - # ???? - postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - ``` + ```ruby + # Disable all components except PostgreSQL + roles ['postgres_role'] + repmgr['enable'] = false + consul['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_monitor['enable'] = false + + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + # ???? + postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + ``` - NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 + NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 1. [Reconfigure GitLab] for the changes to take effect. 1. Note the PostgreSQL node's IP address or hostname, port, and @@ -194,9 +194,9 @@ When using default setup, minimum configuration requires: - `CONSUL_PASSWORD_HASH`. This is a hash generated out of consul username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME + ``` - `CONSUL_SERVER_NODES`. The IP addresses or DNS records of the Consul server nodes. @@ -237,9 +237,9 @@ We will need the following password information for the application's database u - `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME + ``` ##### Pgbouncer information @@ -250,9 +250,9 @@ When using default setup, minimum configuration requires: - `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of pgbouncer username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME + ``` - `PGBOUNCER_NODE`, is the IP address or a FQDN of the node running Pgbouncer. @@ -288,7 +288,6 @@ Make sure you install the necessary dependencies from step 1, add GitLab package repository from step 2. When installing the GitLab package, do not supply `EXTERNAL_URL` value. - #### Configuring the Database nodes 1. Make sure to [configure the Consul nodes](consul.md). @@ -296,53 +295,54 @@ When installing the GitLab package, do not supply `EXTERNAL_URL` value. 1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - # Replace X with value of number of db nodes + 1 - postgresql['max_wal_senders'] = X - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) - - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - > `postgres_role` was introduced with GitLab 10.3 + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + # Replace X with value of number of db nodes + 1 + postgresql['max_wal_senders'] = X + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) + + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration + ``` + + > `postgres_role` was introduced with GitLab 10.3 1. On secondary nodes, add all the configuration specified above for primary node to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration to inform gitlab-ctl that they are standby nodes initially and it need not attempt to register them as primary node + ``` # HA setting to specify if a node should attempt to be master on initialization repmgr['master_on_initialization'] = false @@ -367,31 +367,31 @@ Select one node as a primary node. 1. Open a database prompt: - ```sh - gitlab-psql -d gitlabhq_production - ``` + ```sh + gitlab-psql -d gitlabhq_production + ``` 1. Enable the `pg_trgm` extension: - ```sh - CREATE EXTENSION pg_trgm; - ``` + ```sh + CREATE EXTENSION pg_trgm; + ``` 1. Exit the database prompt by typing `\q` and Enter. 1. Verify the cluster is initialized with one node: - ```sh - gitlab-ctl repmgr cluster show - ``` + ```sh + gitlab-ctl repmgr cluster show + ``` - The output should be similar to the following: + The output should be similar to the following: - ``` - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` + ``` + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` 1. Note down the hostname/ip in the connection string: `host=HOSTNAME`. We will refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value @@ -402,43 +402,43 @@ Select one node as a primary node. 1. Set up the repmgr standby: - ```sh - gitlab-ctl repmgr standby setup MASTER_NODE_NAME - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - # gitlab-ctl repmgr standby setup MASTER_NODE_NAME - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` + ```sh + gitlab-ctl repmgr standby setup MASTER_NODE_NAME + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + # gitlab-ctl repmgr standby setup MASTER_NODE_NAME + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` 1. Verify the node now appears in the cluster: - ```sh - gitlab-ctl repmgr cluster show - ``` + ```sh + gitlab-ctl repmgr cluster show + ``` - The output should be similar to the following: + The output should be similar to the following: - ``` - Role | Name | Upstream | Connection String - ----------+---------|-----------|------------------------------------------------ - * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` + ``` + Role | Name | Upstream | Connection String + ----------+---------|-----------|------------------------------------------------ + * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` Repeat the above steps on all secondary nodes. @@ -478,88 +478,7 @@ Check the [Troubleshooting section](#troubleshooting) before proceeding. #### Configuring the Pgbouncer node -1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`CONSUL_PASSWORD_HASH`](#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information) before executing the next step. - -1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - - ```ruby - # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] - - # Configure Pgbouncer - pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) - - # Configure Consul agent - consul['watchers'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # Replace CONSUL_PASSWORD_HASH with with a generated md5 value - # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value - pgbouncer['users'] = { - 'gitlab-consul': { - password: 'CONSUL_PASSWORD_HASH' - }, - 'pgbouncer': { - password: 'PGBOUNCER_PASSWORD_HASH' - } - } - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - > `pgbouncer_role` was introduced with GitLab 10.3 - -1. [Reconfigure GitLab] for the changes to take effect. - -1. Create a `.pgpass` file so Consul is able to - reload pgbouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked: - - ```sh - gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul - ``` - -##### PGBouncer Checkpoint - -1. Ensure the node is talking to the current master: - - ```sh - gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD - ``` - - If there is an error `psql: ERROR: Auth failed` after typing in the - password, ensure you previously generated the MD5 password hashes with the correct - format. The correct format is to concatenate the password and the username: - `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text - needed to generate an MD5 password hash for the `pgbouncer` user. - -1. Once the console prompt is available, run the following queries: - - ```sh - show databases ; show clients ; - ``` - - The output should be similar to the following: - - ``` - name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections - ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- - gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 - pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 - (2 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls - ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- - C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | - (2 rows) - ``` +See our [documentation for Pgbouncer](pgbouncer.md) for information on running Pgbouncer as part of an HA setup. #### Configuring the Application nodes @@ -568,15 +487,15 @@ attributes set, but the following need to be set. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - # Disable PostgreSQL on the application node - postgresql['enable'] = false + ```ruby + # Disable PostgreSQL on the application node + postgresql['enable'] = false - gitlab_rails['db_host'] = 'PGBOUNCER_NODE' - gitlab_rails['db_port'] = 6432 - gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' - gitlab_rails['auto_migrate'] = false - ``` + gitlab_rails['db_host'] = 'PGBOUNCER_NODE' + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -736,45 +655,45 @@ After deploying the configuration follow these steps: 1. On `10.6.0.21`, our primary database - Enable the `pg_trgm` extension + Enable the `pg_trgm` extension - ```sh - gitlab-psql -d gitlabhq_production - ``` + ```sh + gitlab-psql -d gitlabhq_production + ``` - ``` - CREATE EXTENSION pg_trgm; - ``` + ``` + CREATE EXTENSION pg_trgm; + ``` 1. On `10.6.0.22`, our first standby database - Make this node a standby of the primary + Make this node a standby of the primary - ```sh - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` 1. On `10.6.0.23`, our second standby database - Make this node a standby of the primary + Make this node a standby of the primary - ```sh - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` 1. On `10.6.0.31`, our application server - Set gitlab-consul's pgbouncer password to `toomanysecrets` + Set gitlab-consul's pgbouncer password to `toomanysecrets` - ```sh - gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul - ``` + ```sh + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` - Run database migrations + Run database migrations - ```sh - gitlab-rake gitlab:db:configure - ``` + ```sh + gitlab-rake gitlab:db:configure + ``` #### Example minimal setup @@ -911,16 +830,16 @@ standby nodes. 1. Ensure the old master node is not still active. 1. Login to the server that should become the new master and run: - ```sh - gitlab-ctl repmgr standby promote - ``` + ```sh + gitlab-ctl repmgr standby promote + ``` 1. If there are any other standby servers in the cluster, have them follow the new master server: - ```sh - gitlab-ctl repmgr standby follow NEW_MASTER - ``` + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + ``` #### Restore procedure @@ -930,42 +849,42 @@ after it has been restored to service. - If you want to remove the node from the cluster, on any other node in the cluster, run: - ```sh - gitlab-ctl repmgr standby unregister --node=X - ``` + ```sh + gitlab-ctl repmgr standby unregister --node=X + ``` - where X is the value of node in `repmgr.conf` on the old server. + where X is the value of node in `repmgr.conf` on the old server. - To find this, you can use: + To find this, you can use: - ```sh - awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf - ``` + ```sh + awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf + ``` - It will output something like: + It will output something like: - ``` - 959789412 - ``` + ``` + 959789412 + ``` - Then you will use this id to unregister the node: + Then you will use this id to unregister the node: - ```sh - gitlab-ctl repmgr standby unregister --node=959789412 - ``` + ```sh + gitlab-ctl repmgr standby unregister --node=959789412 + ``` - To add the node as a standby server: - ```sh - gitlab-ctl repmgr standby follow NEW_MASTER - gitlab-ctl restart repmgrd - ``` + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + gitlab-ctl restart repmgrd + ``` - CAUTION: **Warning:** When the server is brought back online, and before - you switch it to a standby node, repmgr will report that there are two masters. - If there are any clients that are still attempting to write to the old master, - this will cause a split, and the old master will need to be resynced from - scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. + CAUTION: **Warning:** When the server is brought back online, and before + you switch it to a standby node, repmgr will report that there are two masters. + If there are any clients that are still attempting to write to the old master, + this will cause a split, and the old master will need to be resynced from + scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. #### Alternate configurations @@ -1008,13 +927,13 @@ the previous section: 1. On the current master node, create a password for the `gitlab` and `gitlab_repmgr` user: - ```sh - gitlab-psql -d template1 - template1=# \password gitlab_repmgr - Enter password: **** - Confirm password: **** - template1=# \password gitlab - ``` + ```sh + gitlab-psql -d template1 + template1=# \password gitlab_repmgr + Enter password: **** + Confirm password: **** + template1=# \password gitlab + ``` 1. On each database node: @@ -1028,9 +947,9 @@ the previous section: 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to when asked: - ```sh - gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' - ``` + ```sh + gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' + ``` 1. On each pgbouncer node, edit `/etc/gitlab/gitlab.rb`: 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for @@ -1058,7 +977,7 @@ If you enable Monitoring, it must be enabled on **all** database servers. ## Troubleshooting -### Consul and PostgreSQL changes not taking effect. +### Consul and PostgreSQL changes not taking effect Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index 9b1b7142e83..83838928519 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -7,33 +7,33 @@ 1. If necessary, install the NFS client utility packages using the following commands: - ``` - # Ubuntu/Debian - apt-get install nfs-common + ``` + # Ubuntu/Debian + apt-get install nfs-common - # CentOS/Red Hat - yum install nfs-utils nfs-utils-lib - ``` + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` 1. Specify the necessary NFS shares. Mounts are specified in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose to configure your NFS server. See [NFS documentation](nfs.md) for the various options. Here is an example snippet to add to `/etc/fstab`: - ``` - 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - ``` + ``` + 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + ``` 1. Create the shared directories. These may be different depending on your NFS mount locations. - ``` - mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data - ``` + ``` + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` 1. Download/install GitLab Omnibus using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other @@ -46,52 +46,52 @@ added NFS mounts in the default data locations. Additionally the UID and GIDs given are just examples and you should configure with your preferred values. - ```ruby - external_url 'https://gitlab.example.com' - - # Prevent GitLab from starting if NFS data mounts are not available - high_availability['mountpoint'] = '/var/opt/gitlab/git-data' - - # Disable components that will not be on the GitLab application server - roles ['application_role'] - nginx['enable'] = true - - # PostgreSQL connection details - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server - gitlab_rails['db_password'] = 'DB password' - - # Redis connection details - gitlab_rails['redis_port'] = '6379' - gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server - gitlab_rails['redis_password'] = 'Redis Password' - - # Ensure UIDs and GIDs match between servers for permissions via NFS - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` + ```ruby + external_url 'https://gitlab.example.com' + + # Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + # Disable components that will not be on the GitLab application server + roles ['application_role'] + nginx['enable'] = true + + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + + # Redis connection details + gitlab_rails['redis_port'] = '6379' + gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server + gitlab_rails['redis_password'] = 'Redis Password' + + # Ensure UIDs and GIDs match between servers for permissions via NFS + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` 1. [Enable monitoring](#enable-monitoring) - > **Note:** To maintain uniformity of links across HA clusters, the `external_url` - on the first application server as well as the additional application - servers should point to the external url that users will use to access GitLab. - In a typical HA setup, this will be the url of the load balancer which will - route traffic to all GitLab application servers in the HA cluster. - > - > **Note:** When you specify `https` in the `external_url`, as in the example - above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If - certificates are not present, Nginx will fail to start. See - [Nginx documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) - for more information. - > - > **Note:** It is best to set the `uid` and `gid`s prior to the initial reconfigure - of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. + > **Note:** To maintain uniformity of links across HA clusters, the `external_url` + on the first application server as well as the additional application + servers should point to the external url that users will use to access GitLab. + In a typical HA setup, this will be the url of the load balancer which will + route traffic to all GitLab application servers in the HA cluster. + > + > **Note:** When you specify `https` in the `external_url`, as in the example + above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If + certificates are not present, Nginx will fail to start. See + [Nginx documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) + for more information. + > + > **Note:** It is best to set the `uid` and `gid`s prior to the initial reconfigure + of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. ## First GitLab application server @@ -114,12 +114,12 @@ need some extra configuration. secondary servers **prior to** running the first `reconfigure` in the steps above. - ```ruby - gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' - gitlab_rails['otp_key_base'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' - gitlab_rails['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' - gitlab_rails['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' - ``` + ```ruby + gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' + gitlab_rails['otp_key_base'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' + gitlab_rails['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' + gitlab_rails['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' + ``` 1. Run `touch /etc/gitlab/skip-auto-reconfigure` to prevent database migrations from running on upgrade. Only the primary GitLab application server should diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 385e7441ac9..cbc1d4bcd52 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -4,7 +4,7 @@ ## Standalone Monitoring node using GitLab Omnibus -The GitLab Omnibus package can be used to configure a standalone Monitoring node running Prometheus and Grafana. +The GitLab Omnibus package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). The monitoring node is not highly available. See [Scaling and High Availability](README.md) for an overview of GitLab scaling and high availability options. @@ -20,44 +20,44 @@ Omnibus: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - external_url 'http://gitlab.example.com' - - # Enable Prometheus - prometheus['enable'] = true - prometheus['listen_address'] = '0.0.0.0:9090' - prometheus['monitor_kubernetes'] = false - - # Enable Grafana - grafana['enable'] = true - grafana['admin_password'] = 'toomanysecrets' - - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Disable all other services - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_monitor['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - ``` + ```ruby + external_url 'http://gitlab.example.com' + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = 'toomanysecrets' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_monitor['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + ``` 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index 561ba214686..6ab6b8bed30 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -42,8 +42,8 @@ maintaining ID mapping without LDAP, in most cases you should enable numeric UID and GIDs (which is off by default in some cases) for simplified permission management between systems: - - [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) - - For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) +- [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) +- For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) ### Improving NFS performance with GitLab @@ -87,10 +87,10 @@ on an Linux NFS server, do the following: 1. On the NFS server, run: - ```sh - echo 0 > /proc/sys/fs/leases-enable - sysctl -w fs.leases-enable=0 - ``` + ```sh + echo 0 > /proc/sys/fs/leases-enable + sysctl -w fs.leases-enable=0 + ``` 1. Restart the NFS server process. For example, on CentOS run `service nfs restart`. diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index 053dae25823..6890b0f7db7 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -14,7 +14,88 @@ It is recommended to run pgbouncer alongside the `gitlab-rails` service, or on i ### Running Pgbouncer as part of an HA GitLab installation -See our [HA documentation for PostgreSQL](database.md) for information on running pgbouncer as part of a HA setup +1. Make sure you collect [`CONSUL_SERVER_NODES`](database.md#consul-information), [`CONSUL_PASSWORD_HASH`](database.md#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](database.md#pgbouncer-information) before executing the next step. + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure Pgbouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # Replace CONSUL_PASSWORD_HASH with with a generated md5 value + # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value + pgbouncer['users'] = { + 'gitlab-consul': { + password: 'CONSUL_PASSWORD_HASH' + }, + 'pgbouncer': { + password: 'PGBOUNCER_PASSWORD_HASH' + } + } + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration + ``` + + > `pgbouncer_role` was introduced with GitLab 10.3 + +1. Run `gitlab-ctl reconfigure` + +1. Create a `.pgpass` file so Consul is able to + reload pgbouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked: + + ```sh + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +#### PGBouncer Checkpoint + +1. Ensure the node is talking to the current master: + + ```sh + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + + If there is an error `psql: ERROR: Auth failed` after typing in the + password, ensure you previously generated the MD5 password hashes with the correct + format. The correct format is to concatenate the password and the username: + `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text + needed to generate an MD5 password hash for the `pgbouncer` user. + +1. Once the console prompt is available, run the following queries: + + ```sh + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ``` + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` ### Running Pgbouncer as part of a non-HA GitLab installation @@ -24,39 +105,39 @@ See our [HA documentation for PostgreSQL](database.md) for information on runnin 1. On your database node, ensure the following is set in your `/etc/gitlab/gitlab.rb` - ```ruby - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' - postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' - postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on - postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node - ``` + ```ruby + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' + postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' + postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on + postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node + ``` 1. Run `gitlab-ctl reconfigure` - **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. + **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. 1. On the node you are running pgbouncer on, make sure the following is set in `/etc/gitlab/gitlab.rb` - ```ruby - pgbouncer['enable'] = true - pgbouncer['databases'] = { - gitlabhq_production: { - host: 'DATABASE_HOST', - user: 'pgbouncer', - password: 'PGBOUNCER_USER_PASSWORD_HASH' - } - } - ``` + ```ruby + pgbouncer['enable'] = true + pgbouncer['databases'] = { + gitlabhq_production: { + host: 'DATABASE_HOST', + user: 'pgbouncer', + password: 'PGBOUNCER_USER_PASSWORD_HASH' + } + } + ``` 1. Run `gitlab-ctl reconfigure` 1. On the node running unicorn, make sure the following is set in `/etc/gitlab/gitlab.rb` - ```ruby - gitlab_rails['db_host'] = 'PGBOUNCER_HOST' - gitlab_rails['db_port'] = '6432' - gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' - ``` + ```ruby + gitlab_rails['db_host'] = 'PGBOUNCER_HOST' + gitlab_rails['db_port'] = '6432' + gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' + ``` 1. Run `gitlab-ctl reconfigure` @@ -66,28 +147,28 @@ See our [HA documentation for PostgreSQL](database.md) for information on runnin > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. - If you enable Monitoring, it must be enabled on **all** pgbouncer servers. +If you enable Monitoring, it must be enabled on **all** pgbouncer servers. - 1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: +1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true + ```ruby + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' - ``` + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' + ``` - 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. ### Interacting with pgbouncer @@ -109,6 +190,7 @@ pgbouncer=# The password you will be prompted for is the PGBOUNCER_USER_PASSWORD To get some basic information about the instance, run + ```shell pgbouncer=# show databases; show clients; show servers; name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index 27310c59755..c29514ed9f6 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -47,28 +47,28 @@ Omnibus: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - ## Enable Redis - redis['enable'] = true - - ## Disable all other services - sidekiq['enable'] = false - gitlab_workhorse['enable'] = false - unicorn['enable'] = false - postgresql['enable'] = false - nginx['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - gitlab_monitor['enable'] = false - gitaly['enable'] = false - - redis['bind'] = '0.0.0.0' - redis['port'] = '6379' - redis['password'] = 'SECRET_PASSWORD_HERE' - - gitlab_rails['auto_migrate'] = false - ``` + ```ruby + ## Enable Redis + redis['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + unicorn['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + gitlab_monitor['enable'] = false + gitaly['enable'] = false + + redis['bind'] = '0.0.0.0' + redis['port'] = '6379' + redis['password'] = 'SECRET_PASSWORD_HERE' + + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Note the Redis node's IP address or hostname, port, and @@ -359,37 +359,37 @@ The prerequisites for a HA Redis setup are the following: 1. SSH into the **master** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.1' + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.1' - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 - # Set up password authentication for Redis (use the same password in all nodes). - redis['password'] = 'redis-password-goes-here' - ``` + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'redis-password-goes-here' + ``` 1. Only the primary GitLab application server should handle migrations. To prevent database migrations from running on upgrade, add the following configuration to your `/etc/gitlab/gitlab.rb` file: - ``` - gitlab_rails['auto_migrate'] = false - ``` + ``` + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. @@ -402,42 +402,42 @@ The prerequisites for a HA Redis setup are the following: 1. SSH into the **slave** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - # Specify server role as 'redis_slave_role' - roles ['redis_slave_role'] + ```ruby + # Specify server role as 'redis_slave_role' + roles ['redis_slave_role'] - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.2' + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.2' - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 - # The same password for Redis authentication you set up for the master node. - redis['password'] = 'redis-password-goes-here' + # The same password for Redis authentication you set up for the master node. + redis['password'] = 'redis-password-goes-here' - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' + # The IP of the master Redis node. + redis['master_ip'] = '10.0.0.1' - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - ``` + # Port of master Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + ``` 1. To prevent reconfigure from running automatically on upgrade, run: - ``` - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` + ``` + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Go through the steps again for all the other slave nodes. @@ -487,89 +487,89 @@ multiple machines with the Sentinel daemon. 1. **You can omit this step if the Sentinels will be hosted in the same node as the other Redis instances.** - [Download/install](https://about.gitlab.com/downloads-ee) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. + [Download/install](https://about.gitlab.com/downloads-ee) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents (if you are installing the Sentinels in the same node as the other Redis instances, some values might be duplicate below): - ```ruby - roles ['redis_sentinel_role'] - - # Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - # The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' - - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 - - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - - ## Configure Sentinel - sentinel['bind'] = '10.0.0.1' - - # Port that Sentinel listens on, uncomment to change to non default. Defaults - # to `26379`. - # sentinel['port'] = 26379 - - ## Quorum must reflect the amount of voting sentinels it take to start a failover. - ## Value must NOT be greater then the amount of sentinels. - ## - ## The quorum can be used to tune Sentinel in two ways: - ## 1. If a the quorum is set to a value smaller than the majority of Sentinels - ## we deploy, we are basically making Sentinel more sensible to master failures, - ## triggering a failover as soon as even just a minority of Sentinels is no longer - ## able to talk with the master. - ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are - ## making Sentinel able to failover only when there are a very large number (larger - ## than majority) of well connected Sentinels which agree about the master being down.s - sentinel['quorum'] = 2 - - ## Consider unresponsive server down after x amount of ms. - # sentinel['down_after_milliseconds'] = 10000 - - ## Specifies the failover timeout in milliseconds. It is used in many ways: - ## - ## - The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## - The time needed for a slave replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## - The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). - ## - ## - The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - # sentinel['failover_timeout'] = 60000 - ``` + ```ruby + roles ['redis_sentinel_role'] + + # Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + # The same password for Redis authentication you set up for the master node. + redis['master_password'] = 'redis-password-goes-here' + + # The IP of the master Redis node. + redis['master_ip'] = '10.0.0.1' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Port of master Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel + sentinel['bind'] = '10.0.0.1' + + # Port that Sentinel listens on, uncomment to change to non default. Defaults + # to `26379`. + # sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to master failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the master. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the master being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + # sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same master by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a slave replicating to a wrong master according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right master, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (SLAVEOF NO ONE yet not + ## acknowledged by the promoted slave). + ## + ## - The maximum time a failover in progress waits for all the slaves to be + ## reconfigured as slaves of the new master. However even after this time + ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + # sentinel['failover_timeout'] = 60000 + ``` 1. To prevent database migrations from running on upgrade, run: - ``` - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` + ``` + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` - Only the primary GitLab application server should handle migrations. + Only the primary GitLab application server should handle migrations. 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Go through the steps again for all the other Sentinel nodes. @@ -593,20 +593,20 @@ which ideally should not have Redis or Sentinels on it for a HA setup. 1. SSH into the server where the GitLab application is installed. 1. Edit `/etc/gitlab/gitlab.rb` and add/change the following lines: - ``` - ## Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - ## The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - ## A list of sentinels with `host` and `port` - gitlab_rails['redis_sentinels'] = [ - {'host' => '10.0.0.1', 'port' => 26379}, - {'host' => '10.0.0.2', 'port' => 26379}, - {'host' => '10.0.0.3', 'port' => 26379} - ] - ``` + ```ruby + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = 'redis-password-goes-here' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.0.0.1', 'port' => 26379}, + {'host' => '10.0.0.2', 'port' => 26379}, + {'host' => '10.0.0.3', 'port' => 26379} + ] + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. @@ -791,31 +791,34 @@ cache, queues, and shared_state. To make this work with Sentinel: 1. Set the appropriate variable in `/etc/gitlab/gitlab.rb` for each instance you are using: - ```ruby - gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL - gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL - gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL - ``` + ```ruby + gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL + gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL + gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL + ``` + **Note**: Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_MASTER_NAME` - 1. PASSWORD is the plaintext password for the Redis instance - 1. SENTINEL_MASTER_NAME is the Sentinel master name (e.g. `gitlab-redis-cache`) + 1. PASSWORD is the plaintext password for the Redis instance + 1. SENTINEL_MASTER_NAME is the Sentinel master name (e.g. `gitlab-redis-cache`) + 1. Include an array of hashes with host/port combinations, such as the following: - ```ruby - gitlab_rails['redis_cache_sentinels'] = [ - { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_queues_sentinels'] = [ - { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, - { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } - ] - ``` + ```ruby + gitlab_rails['redis_cache_sentinels'] = [ + { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_queues_sentinels'] = [ + { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, + { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } + ] + ``` + 1. Note that for each persistence class, GitLab will default to using the configuration specified in `gitlab_rails['redis_sentinels']` unless overridden by the settings above. @@ -879,12 +882,12 @@ in order for the HA setup to work as expected. Before proceeding with the troubleshooting below, check your firewall rules: - Redis machines - - Accept TCP connection in `6379` - - Connect to the other Redis machines via TCP in `6379` + - Accept TCP connection in `6379` + - Connect to the other Redis machines via TCP in `6379` - Sentinel machines - - Accept TCP connection in `26379` - - Connect to other Sentinel machines via TCP in `26379` - - Connect to the Redis machines via TCP in `6379` + - Accept TCP connection in `26379` + - Connect to other Sentinel machines via TCP in `26379` + - Connect to the Redis machines via TCP in `6379` ### Troubleshooting Redis replication @@ -952,38 +955,38 @@ To make sure your configuration is correct: 1. SSH into your GitLab application server 1. Enter the Rails console: - ``` - # For Omnibus installations - sudo gitlab-rails console + ``` + # For Omnibus installations + sudo gitlab-rails console - # For source installations - sudo -u git rails console production - ``` + # For source installations + sudo -u git rails console production + ``` 1. Run in the console: - ```ruby - redis = Redis.new(Gitlab::Redis::SharedState.params) - redis.info - ``` + ```ruby + redis = Redis.new(Gitlab::Redis::SharedState.params) + redis.info + ``` - Keep this screen open and try to simulate a failover below. + Keep this screen open and try to simulate a failover below. 1. To simulate a failover on master Redis, SSH into the Redis server and run: - ```bash - # port must match your master redis port, and the sleep time must be a few seconds bigger than defined one - redis-cli -h localhost -p 6379 DEBUG sleep 20 - ``` + ```bash + # port must match your master redis port, and the sleep time must be a few seconds bigger than defined one + redis-cli -h localhost -p 6379 DEBUG sleep 20 + ``` 1. Then back in the Rails console from the first step, run: - ``` - redis.info - ``` + ``` + redis.info + ``` - You should see a different port after a few seconds delay - (the failover/reconnect time). + You should see a different port after a few seconds delay + (the failover/reconnect time). ## Changelog diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index be6b547372a..a5463e5128c 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -49,22 +49,22 @@ Assuming that the Redis master instance IP is `10.0.0.1`: 1. [Install Redis](../../install/installation.md#7-redis). 1. Edit `/etc/redis/redis.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` 1. Restart the Redis service for the changes to take effect. @@ -75,25 +75,25 @@ Assuming that the Redis slave instance IP is `10.0.0.2`: 1. [Install Redis](../../install/installation.md#7-redis). 1. Edit `/etc/redis/redis.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.2 + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.2 - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here - ## Define `slaveof` pointing to the Redis master instance with IP and port. - slaveof 10.0.0.1 6379 - ``` + ## Define `slaveof` pointing to the Redis master instance with IP and port. + slaveof 10.0.0.1 6379 + ``` 1. Restart the Redis service for the changes to take effect. 1. Go through the steps again for all the other slave nodes. @@ -110,56 +110,57 @@ master with IP `10.0.0.1` (some settings might overlap with the master): 1. [Install Redis Sentinel](https://redis.io/topics/sentinel) 1. Edit `/etc/redis/sentinel.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force Sentinel to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 26379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - - ## Define with `sentinel auth-pass` the same shared password you have - ## defined for both Redis master and slaves instances. - sentinel auth-pass gitlab-redis redis-password-goes-here - - ## Define with `sentinel monitor` the IP and port of the Redis - ## master node, and the quorum required to start a failover. - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - - ## Define with `sentinel down-after-milliseconds` the time in `ms` - ## that an unresponsive server will be considered down. - sentinel down-after-milliseconds gitlab-redis 10000 - - ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple - ## meanings: - ## - ## * The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## * The time needed for a slave replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## * The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). - ## - ## * The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - sentinel failover_timeout 30000 - ``` + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force Sentinel to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 26379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + + ## Define with `sentinel auth-pass` the same shared password you have + ## defined for both Redis master and slaves instances. + sentinel auth-pass gitlab-redis redis-password-goes-here + + ## Define with `sentinel monitor` the IP and port of the Redis + ## master node, and the quorum required to start a failover. + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + + ## Define with `sentinel down-after-milliseconds` the time in `ms` + ## that an unresponsive server will be considered down. + sentinel down-after-milliseconds gitlab-redis 10000 + + ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple + ## meanings: + ## + ## * The time needed to re-start a failover after a previous failover was + ## already tried against the same master by a given Sentinel, is two + ## times the failover timeout. + ## + ## * The time needed for a slave replicating to a wrong master according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right master, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## * The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (SLAVEOF NO ONE yet not + ## acknowledged by the promoted slave). + ## + ## * The maximum time a failover in progress waits for all the slaves to be + ## reconfigured as slaves of the new master. However even after this time + ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + sentinel failover_timeout 30000 + ``` + 1. Restart the Redis service for the changes to take effect. 1. Go through the steps again for all the other Sentinel nodes. @@ -180,21 +181,21 @@ setup: [resque.yml.example][resque], and uncomment the Sentinel lines, pointing to the correct server credentials: - ```yaml - # resque.yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` + ```yaml + # resque.yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -232,23 +233,23 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.1 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` + ```conf + bind 10.0.0.1 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.1 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.1 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -256,24 +257,24 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.2 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 - ``` + ```conf + bind 10.0.0.2 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + slaveof 10.0.0.1 6379 + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.2 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.2 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -281,24 +282,24 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.3 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 - ``` + ```conf + bind 10.0.0.3 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + slaveof 10.0.0.1 6379 + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.3 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.3 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -306,20 +307,20 @@ or a failover promotes a different **Master** node. 1. Edit `/home/git/gitlab/config/resque.yml`: - ```yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` + ```yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` 1. [Restart GitLab][restart] for the changes to take effect. diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 8de7b0bc57e..c2ac063ce37 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -1,6 +1,7 @@ # PlantUML & GitLab -> [Introduced][ce-8537] in GitLab 8.16. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8537) in +> GitLab 8.16. When [PlantUML](http://plantuml.com) integration is enabled and configured in GitLab we are able to create simple diagrams in AsciiDoc and Markdown documents @@ -15,7 +16,9 @@ server that will generate the diagrams. With Docker, you can just run a container like this: -`docker run -d --name plantuml -p 8080:8080 plantuml/plantuml-server:tomcat` +```sh +docker run -d --name plantuml -p 8080:8080 plantuml/plantuml-server:tomcat +``` The **PlantUML URL** will be the hostname of the server running the container. @@ -26,7 +29,7 @@ own PlantUML server is easy in Debian/Ubuntu distributions using Tomcat. First you need to create a `plantuml.war` file from the source code: -``` +```sh sudo apt-get install graphviz openjdk-8-jdk git-core maven git clone https://github.com/plantuml/plantuml-server.git cd plantuml-server @@ -36,7 +39,7 @@ mvn package The above sequence of commands will generate a WAR file that can be deployed using Tomcat: -``` +```sh sudo apt-get install tomcat7 sudo cp target/plantuml.war /var/lib/tomcat7/webapps/plantuml.war sudo chown tomcat7:tomcat7 /var/lib/tomcat7/webapps/plantuml.war @@ -46,7 +49,7 @@ sudo service tomcat7 restart Once the Tomcat service restarts the PlantUML service will be ready and listening for requests on port 8080: -``` +```text http://localhost:8080/plantuml ``` @@ -57,9 +60,10 @@ you can change these defaults by editing the `/etc/tomcat7/server.xml` file. You need to enable PlantUML integration from Settings under Admin Area. To do that, login with an Admin account and do following: - - in GitLab go to **Admin Area**->**Settings**->**Integrations**->**PlantUML** - - check **Enable PlantUML** checkbox - - set the PlantUML instance as **PlantUML URL** +- In GitLab, go to **Admin Area > Settings > Integrations**. +- Expand the **PlantUML** section. +- Check **Enable PlantUML** checkbox. +- Set the PlantUML instance as **PlantUML URL**. ## Creating Diagrams @@ -68,33 +72,34 @@ our AsciiDoc snippets, wikis and repos using delimited blocks: - **Markdown** - <pre> - ```plantuml - Bob -> Alice : hello - Alice -> Bob : Go Away - ```</pre> + ````markdown + ```plantuml + Bob -> Alice : hello + Alice -> Bob : Go Away + ``` + ```` - **AsciiDoc** - ``` - [plantuml, format="png", id="myDiagram", width="200px"] - ---- - Bob->Alice : hello - Alice -> Bob : Go Away - ---- - ``` + ```text + [plantuml, format="png", id="myDiagram", width="200px"] + ---- + Bob->Alice : hello + Alice -> Bob : Go Away + ---- + ``` - **reStructuredText** - ``` - .. plantuml:: - :caption: Caption with **bold** and *italic* + ```text + .. plantuml:: + :caption: Caption with **bold** and *italic* - Bob -> Alice: hello - Alice -> Bob: Go Away - ``` + Bob -> Alice: hello + Alice -> Bob: Go Away + ``` - You can also use the `uml::` directive for compatibility with [sphinxcontrib-plantuml](https://pypi.org/project/sphinxcontrib-plantuml/), but please note that we currently only support the `caption` option. + You can also use the `uml::` directive for compatibility with [sphinxcontrib-plantuml](https://pypi.org/project/sphinxcontrib-plantuml/), but please note that we currently only support the `caption` option. The above blocks will be converted to an HTML img tag with source pointing to the PlantUML instance. If the PlantUML server is correctly configured, this should @@ -111,12 +116,10 @@ diagram delimiters `@startuml`/`@enduml` as these are replaced by the AsciiDoc ` Some parameters can be added to the AsciiDoc block definition: - - *format*: Can be either `png` or `svg`. Note that `svg` is not supported by - all browsers so use with care. The default is `png`. - - *id*: A CSS id added to the diagram HTML tag. - - *width*: Width attribute added to the img tag. - - *height*: Height attribute added to the img tag. +- *format*: Can be either `png` or `svg`. Note that `svg` is not supported by + all browsers so use with care. The default is `png`. +- *id*: A CSS id added to the diagram HTML tag. +- *width*: Width attribute added to the img tag. +- *height*: Height attribute added to the img tag. Markdown does not support any parameters and will always use PNG format. - -[ce-8537]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8537 diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index c34858cd0db..24c9cc0bea9 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -41,9 +41,9 @@ detail below. - Every session URL that is created has an authorization header that needs to be sent, to establish a `wss` connection. - The session URL is not exposed to the users in any way. GitLab holds all the state internally and proxies accordingly. -## Enabling and disabling terminal support +## Enabling and disabling terminal support -NOTE: **Note:** AWS Elastic Load Balancers (ELBs) do not support web sockets. +NOTE: **Note:** AWS Elastic Load Balancers (ELBs) do not support web sockets. AWS Application Load Balancers (ALBs) must be used if you want web terminals to work. See [AWS Elastic Load Balancing Product Comparison](https://aws.amazon.com/elasticloadbalancing/features/#compare) for more information. diff --git a/doc/administration/job_traces.md b/doc/administration/job_traces.md index 957916893d7..6a06eb240de 100644 --- a/doc/administration/job_traces.md +++ b/doc/administration/job_traces.md @@ -95,9 +95,9 @@ If job traces have already been archived into local storage, and you want to mig 1. Ensure [Object storage integration for Job Artifacts](job_artifacts.md#object-storage-settings) is enabled 1. Execute the following command - ```bash - gitlab-rake gitlab:traces:migrate - ``` + ```bash + gitlab-rake gitlab:traces:migrate + ``` ## How to remove job traces diff --git a/doc/administration/monitoring/ip_whitelist.md b/doc/administration/monitoring/ip_whitelist.md index ad2773de132..6bb2fe81b2c 100644 --- a/doc/administration/monitoring/ip_whitelist.md +++ b/doc/administration/monitoring/ip_whitelist.md @@ -12,9 +12,9 @@ hosts or use IP ranges: 1. Open `/etc/gitlab/gitlab.rb` and add or uncomment the following: - ```ruby - gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] - ``` + ```ruby + gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] + ``` 1. Save the file and [reconfigure] GitLab for the changes to take effect. @@ -24,13 +24,13 @@ hosts or use IP ranges: 1. Edit `config/gitlab.yml`: - ```yaml - monitoring: - # by default only local IPs are allowed to access monitoring resources - ip_whitelist: - - 127.0.0.0/8 - - 192.168.0.1 - ``` + ```yaml + monitoring: + # by default only local IPs are allowed to access monitoring resources + ip_whitelist: + - 127.0.0.0/8 + - 192.168.0.1 + ``` 1. Save the file and [restart] GitLab for the changes to take effect. diff --git a/doc/administration/monitoring/prometheus/gitlab_monitor_exporter.md b/doc/administration/monitoring/prometheus/gitlab_monitor_exporter.md index f68b03d1ade..9aa4dfa5ab7 100644 --- a/doc/administration/monitoring/prometheus/gitlab_monitor_exporter.md +++ b/doc/administration/monitoring/prometheus/gitlab_monitor_exporter.md @@ -12,9 +12,9 @@ To enable the GitLab monitor exporter: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `true`: - ```ruby - gitlab_monitor['enable'] = true - ``` + ```ruby + gitlab_monitor['enable'] = true + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index ce65d77274b..341ea3330d7 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -39,9 +39,9 @@ To disable Prometheus and all of its exporters, as well as any added in the futu 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `false`: - ```ruby - prometheus_monitoring['enable'] = false - ``` + ```ruby + prometheus_monitoring['enable'] = false + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect. @@ -61,19 +61,19 @@ To change the address/port that Prometheus listens on: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line: - ```ruby - prometheus['listen_address'] = 'localhost:9090' - ``` + ```ruby + prometheus['listen_address'] = 'localhost:9090' + ``` - Replace `localhost:9090` with the address/port you want Prometheus to - listen on. If you would like to allow access to Prometheus to hosts other - than `localhost`, leave out the host, or use `0.0.0.0` to allow public access: + Replace `localhost:9090` with the address/port you want Prometheus to + listen on. If you would like to allow access to Prometheus to hosts other + than `localhost`, leave out the host, or use `0.0.0.0` to allow public access: - ```ruby - prometheus['listen_address'] = ':9090' - # or - prometheus['listen_address'] = '0.0.0.0:9090' - ``` + ```ruby + prometheus['listen_address'] = ':9090' + # or + prometheus['listen_address'] = '0.0.0.0:9090' + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect @@ -90,22 +90,22 @@ To use an external Prometheus server: 1. Edit `/etc/gitlab/gitlab.rb`. 1. Disable the bundled Prometheus: - ```ruby - prometheus['enable'] = false - ``` + ```ruby + prometheus['enable'] = false + ``` 1. Set each bundled service's [exporter](#bundled-software-metrics) to listen on a network address, for example: - ```ruby - gitlab_monitor['listen_address'] = '0.0.0.0' - sidekiq['listen_address'] = '0.0.0.0' - gitlab_monitor['listen_port'] = '9168' - node_exporter['listen_address'] = '0.0.0.0:9100' - redis_exporter['listen_address'] = '0.0.0.0:9121' - postgres_exporter['listen_address'] = '0.0.0.0:9187' - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - gitlab_workhorse['prometheus_listen_addr'] = "0.0.0.0:9229" - ``` + ```ruby + gitlab_monitor['listen_address'] = '0.0.0.0' + sidekiq['listen_address'] = '0.0.0.0' + gitlab_monitor['listen_port'] = '9168' + node_exporter['listen_address'] = '0.0.0.0:9100' + redis_exporter['listen_address'] = '0.0.0.0:9121' + postgres_exporter['listen_address'] = '0.0.0.0:9187' + gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + gitlab_workhorse['prometheus_listen_addr'] = "0.0.0.0:9229" + ``` 1. Install and set up a dedicated Prometheus instance, if necessary, using the [official installation instructions](https://prometheus.io/docs/prometheus/latest/installation/). 1. Add the Prometheus server IP address to the [monitoring IP whitelist](../ip_whitelist.html). For example: @@ -117,14 +117,14 @@ To use an external Prometheus server: 1. To scrape nginx metrics, you'll also need to configure nginx to allow the Prometheus server IP. For example: - ```ruby - nginx['status']['options'] = { - "server_tokens" => "off", - "access_log" => "off", - "allow" => "192.168.0.1", - "deny" => "all", - } - ``` + ```ruby + nginx['status']['options'] = { + "server_tokens" => "off", + "access_log" => "off", + "allow" => "192.168.0.1", + "deny" => "all", + } + ``` 1. [Reconfigure GitLab][reconfigure] to apply the changes 1. Edit the Prometheus server's configuration file. @@ -132,17 +132,17 @@ To use an external Prometheus server: [scrape target configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cscrape_config%3E). For example, a sample snippet using `static_configs`: - ```yaml - scrape_configs: - - job_name: 'gitlab_exporters' - static_configs: - - targets: ['1.1.1.1:9168', '1.1.1.1:9236', '1.1.1.1:9236', '1.1.1.1:9100', '1.1.1.1:9121', '1.1.1.1:9187'] + ```yaml + scrape_configs: + - job_name: 'gitlab_exporters' + static_configs: + - targets: ['1.1.1.1:9168', '1.1.1.1:9236', '1.1.1.1:9236', '1.1.1.1:9100', '1.1.1.1:9121', '1.1.1.1:9187'] - - job_name: 'gitlab_metrics' - metrics_path: /-/metrics - static_configs: - - targets: ['1.1.1.1:443'] - ``` + - job_name: 'gitlab_metrics' + metrics_path: /-/metrics + static_configs: + - targets: ['1.1.1.1:443'] + ``` 1. Restart the Prometheus server. @@ -241,9 +241,9 @@ To disable the monitoring of Kubernetes: 1. Edit `/etc/gitlab/gitlab.rb`. 1. Add or find and uncomment the following line and set it to `false`: - ```ruby - prometheus['monitor_kubernetes'] = false - ``` + ```ruby + prometheus['monitor_kubernetes'] = false + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect. diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index aef7758a88f..bcacfaa3be5 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -13,9 +13,9 @@ To enable the node exporter: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `true`: - ```ruby - node_exporter['enable'] = true - ``` + ```ruby + node_exporter['enable'] = true + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index d76834fdbea..6183594c69c 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -12,9 +12,9 @@ To enable the PgBouncer exporter: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `true`: - ```ruby - pgbouncer_exporter['enable'] = true - ``` + ```ruby + pgbouncer_exporter['enable'] = true + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect. diff --git a/doc/administration/monitoring/prometheus/postgres_exporter.md b/doc/administration/monitoring/prometheus/postgres_exporter.md index 8e2d3162f88..3ad15b65497 100644 --- a/doc/administration/monitoring/prometheus/postgres_exporter.md +++ b/doc/administration/monitoring/prometheus/postgres_exporter.md @@ -12,9 +12,9 @@ To enable the postgres exporter: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `true`: - ```ruby - postgres_exporter['enable'] = true - ``` + ```ruby + postgres_exporter['enable'] = true + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index d54d409dbb6..1520115a38d 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -13,9 +13,9 @@ To enable the Redis exporter: 1. Edit `/etc/gitlab/gitlab.rb` 1. Add or find and uncomment the following line, making sure it's set to `true`: - ```ruby - redis_exporter['enable'] = true - ``` + ```ruby + redis_exporter['enable'] = true + ``` 1. Save the file and [reconfigure GitLab][reconfigure] for the changes to take effect diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index b329abdca08..ea69378b249 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -117,81 +117,81 @@ the database. The following instructions can be used to build OpenSSH 7.5: 1. First, download the package and install the required packages: - ``` - sudo su - - cd /tmp - curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz - tar xzvf openssh-7.5p1.tar.gz - yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel - ``` + ``` + sudo su - + cd /tmp + curl --remote-name https://mirrors.evowise.com/pub/OpenBSD/OpenSSH/portable/openssh-7.5p1.tar.gz + tar xzvf openssh-7.5p1.tar.gz + yum install rpm-build gcc make wget openssl-devel krb5-devel pam-devel libX11-devel xmkmf libXt-devel + ``` 1. Prepare the build by copying files to the right place: - ``` - mkdir -p /root/rpmbuild/{SOURCES,SPECS} - cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/ - cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/ - cd /root/rpmbuild/SPECS - ``` + ``` + mkdir -p /root/rpmbuild/{SOURCES,SPECS} + cp ./openssh-7.5p1/contrib/redhat/openssh.spec /root/rpmbuild/SPECS/ + cp openssh-7.5p1.tar.gz /root/rpmbuild/SOURCES/ + cd /root/rpmbuild/SPECS + ``` 1. Next, set the spec settings properly: - ``` - sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec - sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec - sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec - ``` + ``` + sed -i -e "s/%define no_gnome_askpass 0/%define no_gnome_askpass 1/g" openssh.spec + sed -i -e "s/%define no_x11_askpass 0/%define no_x11_askpass 1/g" openssh.spec + sed -i -e "s/BuildPreReq/BuildRequires/g" openssh.spec + ``` 1. Build the RPMs: - ``` - rpmbuild -bb openssh.spec - ``` + ``` + rpmbuild -bb openssh.spec + ``` 1. Ensure the RPMs were built: - ``` - ls -al /root/rpmbuild/RPMS/x86_64/ - ``` + ``` + ls -al /root/rpmbuild/RPMS/x86_64/ + ``` - You should see something as the following: + You should see something as the following: - ``` - total 1324 - drwxr-xr-x. 2 root root 4096 Jun 20 19:37 . - drwxr-xr-x. 3 root root 19 Jun 20 19:37 .. - -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm - -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm - -rw-r--r--. 1 root root 17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm - -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm - ``` + ``` + total 1324 + drwxr-xr-x. 2 root root 4096 Jun 20 19:37 . + drwxr-xr-x. 3 root root 19 Jun 20 19:37 .. + -rw-r--r--. 1 root root 470828 Jun 20 19:37 openssh-7.5p1-1.x86_64.rpm + -rw-r--r--. 1 root root 490716 Jun 20 19:37 openssh-clients-7.5p1-1.x86_64.rpm + -rw-r--r--. 1 root root 17020 Jun 20 19:37 openssh-debuginfo-7.5p1-1.x86_64.rpm + -rw-r--r--. 1 root root 367516 Jun 20 19:37 openssh-server-7.5p1-1.x86_64.rpm + ``` 1. Install the packages. OpenSSH packages will replace `/etc/pam.d/sshd` with its own version, which may prevent users from logging in, so be sure that the file is backed up and restored after installation: - ``` - timestamp=$(date +%s) - cp /etc/pam.d/sshd pam-ssh-conf-$timestamp - rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm - yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd - ``` + ``` + timestamp=$(date +%s) + cp /etc/pam.d/sshd pam-ssh-conf-$timestamp + rpm -Uvh /root/rpmbuild/RPMS/x86_64/*.rpm + yes | cp pam-ssh-conf-$timestamp /etc/pam.d/sshd + ``` 1. Verify the installed version. In another window, attempt to login to the server: - ``` - ssh -v <your-centos-machine> - ``` + ``` + ssh -v <your-centos-machine> + ``` - You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5" + You should see a line that reads: "debug1: Remote protocol version 2.0, remote software version OpenSSH_7.5" - If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`). + If not, you may need to restart sshd (e.g. `systemctl restart sshd.service`). -1. *IMPORTANT!* Open a new SSH session to your server before exiting to make - sure everything is working! If you need to downgrade, simple install the - older package: +1. *IMPORTANT!* Open a new SSH session to your server before exiting to make + sure everything is working! If you need to downgrade, simple install the + older package: - ``` - # Only run this if you run into a problem logging in - yum downgrade openssh-server openssh openssh-clients - ``` + ``` + # Only run this if you run into a problem logging in + yum downgrade openssh-server openssh openssh-clients + ``` diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index c0c242733a2..b5922d9d99d 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -78,34 +78,37 @@ executed, and then read the same 1,000 files. [repository storage path](../repository_storage_paths.md). 1. Create a temporary directory for the test so it's easy to remove the files later: - ```sh - mkdir test; cd test - ``` + ```sh + mkdir test; cd test + ``` + 1. Run the command: - ```sh - time for i in {0..1000}; do echo 'test' > "test${i}.txt"; done - ``` -1. To benchmark read performance, run the command: + ```sh + time for i in {0..1000}; do echo 'test' > "test${i}.txt"; done + ``` - ```sh - time for i in {0..1000}; do cat "test${i}.txt" > /dev/null; done - ``` -1. Remove the test files: +1. To benchmark read performance, run the command: ```sh - cd ../; rm -rf test + time for i in {0..1000}; do cat "test${i}.txt" > /dev/null; done ``` +1. Remove the test files: + + ```sh + cd ../; rm -rf test + ``` + The output of the `time for ...` commands will look similar to the following. The important metric is the `real` time. ```sh $ time for i in {0..1000}; do echo 'test' > "test${i}.txt"; done -real 0m0.116s -user 0m0.025s -sys 0m0.091s +real 0m0.116s +user 0m0.025s +sys 0m0.091s $ time for i in {0..1000}; do cat "test${i}.txt" > /dev/null; done diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 102656b01ec..b5b8f124274 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -124,9 +124,9 @@ The Pages daemon doesn't listen to the outside world. 1. Set the external URL for GitLab Pages in `/etc/gitlab/gitlab.rb`: - ```shell - pages_external_url 'http://example.io' - ``` + ```shell + pages_external_url 'http://example.io' + ``` 1. [Reconfigure GitLab][reconfigure]. @@ -149,16 +149,16 @@ outside world. 1. Place the certificate and key inside `/etc/gitlab/ssl` 1. In `/etc/gitlab/gitlab.rb` specify the following configuration: - ```shell - pages_external_url 'https://example.io' + ```shell + pages_external_url 'https://example.io' - pages_nginx['redirect_http_to_https'] = true - pages_nginx['ssl_certificate'] = "/etc/gitlab/ssl/pages-nginx.crt" - pages_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/pages-nginx.key" - ``` + pages_nginx['redirect_http_to_https'] = true + pages_nginx['ssl_certificate'] = "/etc/gitlab/ssl/pages-nginx.crt" + pages_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/pages-nginx.key" + ``` - where `pages-nginx.crt` and `pages-nginx.key` are the SSL cert and key, - respectively. + where `pages-nginx.crt` and `pages-nginx.key` are the SSL cert and key, + respectively. 1. [Reconfigure GitLab][reconfigure]. @@ -171,9 +171,9 @@ behavior: 1. Edit `/etc/gitlab/gitlab.rb`. 1. Set the `inplace_chroot` to `true` for GitLab Pages: - ```shell - gitlab_pages['inplace_chroot'] = true - ``` + ```shell + gitlab_pages['inplace_chroot'] = true + ``` 1. [Reconfigure GitLab][reconfigure]. @@ -206,16 +206,16 @@ world. Custom domains are supported, but no TLS. 1. Edit `/etc/gitlab/gitlab.rb`: - ```shell - pages_external_url "http://example.io" - nginx['listen_addresses'] = ['192.0.2.1'] - pages_nginx['enable'] = false - gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] - ``` + ```shell + pages_external_url "http://example.io" + nginx['listen_addresses'] = ['192.0.2.1'] + pages_nginx['enable'] = false + gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] + ``` - where `192.0.2.1` is the primary IP address that GitLab is listening to and - `192.0.2.2` and `2001::2` are the secondary IPs the GitLab Pages daemon - listens on. If you don't have IPv6, you can omit the IPv6 address. + where `192.0.2.1` is the primary IP address that GitLab is listening to and + `192.0.2.2` and `2001::2` are the secondary IPs the GitLab Pages daemon + listens on. If you don't have IPv6, you can omit the IPv6 address. 1. [Reconfigure GitLab][reconfigure]. @@ -237,19 +237,19 @@ world. Custom domains and TLS are supported. 1. Edit `/etc/gitlab/gitlab.rb`: - ```shell - pages_external_url "https://example.io" - nginx['listen_addresses'] = ['192.0.2.1'] - pages_nginx['enable'] = false - gitlab_pages['cert'] = "/etc/gitlab/ssl/example.io.crt" - gitlab_pages['cert_key'] = "/etc/gitlab/ssl/example.io.key" - gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] - gitlab_pages['external_https'] = ['192.0.2.2:443', '[2001::2]:443'] - ``` + ```shell + pages_external_url "https://example.io" + nginx['listen_addresses'] = ['192.0.2.1'] + pages_nginx['enable'] = false + gitlab_pages['cert'] = "/etc/gitlab/ssl/example.io.crt" + gitlab_pages['cert_key'] = "/etc/gitlab/ssl/example.io.key" + gitlab_pages['external_http'] = ['192.0.2.2:80', '[2001::2]:80'] + gitlab_pages['external_https'] = ['192.0.2.2:443', '[2001::2]:443'] + ``` - where `192.0.2.1` is the primary IP address that GitLab is listening to and - `192.0.2.2` and `2001::2` are the secondary IPs where the GitLab Pages daemon - listens on. If you don't have IPv6, you can omit the IPv6 address. + where `192.0.2.1` is the primary IP address that GitLab is listening to and + `192.0.2.2` and `2001::2` are the secondary IPs where the GitLab Pages daemon + listens on. If you don't have IPv6, you can omit the IPv6 address. 1. [Reconfigure GitLab][reconfigure]. @@ -287,9 +287,9 @@ Pages access control is disabled by default. To enable it: 1. Enable it in `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_pages['access_control'] = true - ``` + ```ruby + gitlab_pages['access_control'] = true + ``` 1. [Reconfigure GitLab][reconfigure]. 1. Users can now configure it in their [projects' settings](../../user/project/pages/introduction.md#gitlab-pages-access-control-core-only). @@ -302,9 +302,9 @@ pages: 1. Configure in `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_pages['http_proxy'] = 'http://example:8080' - ``` + ```ruby + gitlab_pages['http_proxy'] = 'http://example:8080' + ``` 1. [Reconfigure Gitlab][reconfigure] for the changes to take effect. @@ -319,9 +319,9 @@ Follow the steps below to configure verbose logging of GitLab Pages daemon. If you wish to make it log events with level `DEBUG` you must configure this in `/etc/gitlab/gitlab.rb`: - ```shell - gitlab_pages['log_verbose'] = true - ``` + ```shell + gitlab_pages['log_verbose'] = true + ``` 1. [Reconfigure GitLab][reconfigure]. @@ -334,9 +334,9 @@ are stored. If you wish to store them in another location you must set it up in `/etc/gitlab/gitlab.rb`: - ```shell - gitlab_rails['pages_path'] = "/mnt/storage/pages" - ``` + ```shell + gitlab_rails['pages_path'] = "/mnt/storage/pages" + ``` 1. [Reconfigure GitLab][reconfigure]. @@ -347,19 +347,19 @@ Omnibus GitLab 11.1. 1. By default the listener is configured to listen for requests on `localhost:8090`. - If you wish to disable it you must configure this in - `/etc/gitlab/gitlab.rb`: + If you wish to disable it you must configure this in + `/etc/gitlab/gitlab.rb`: - ```shell - gitlab_pages['listen_proxy'] = nil - ``` + ```shell + gitlab_pages['listen_proxy'] = nil + ``` - If you wish to make it listen on a different port you must configure this also in - `/etc/gitlab/gitlab.rb`: + If you wish to make it listen on a different port you must configure this also in + `/etc/gitlab/gitlab.rb`: - ```shell - gitlab_pages['listen_proxy'] = "localhost:10080" - ``` + ```shell + gitlab_pages['listen_proxy'] = "localhost:10080" + ``` 1. [Reconfigure GitLab][reconfigure]. @@ -381,28 +381,29 @@ Follow the steps below to configure GitLab Pages in a separate server. 1. On `app2` install GitLab omnibus and modify `/etc/gitlab/gitlab.rb` this way: - ```shell - external_url 'http://<ip-address-of-the-server>' - pages_external_url "http://<your-pages-domain>" - postgresql['enable'] = false - redis['enable'] = false - prometheus['enable'] = false - unicorn['enable'] = false - sidekiq['enable'] = false - gitlab_workhorse['enable'] = false - gitaly['enable'] = false - alertmanager['enable'] = false - node_exporter['enable'] = false - gitlab_rails['auto_migrate'] = false - ``` + ```shell + external_url 'http://<ip-address-of-the-server>' + pages_external_url "http://<your-pages-domain>" + postgresql['enable'] = false + redis['enable'] = false + prometheus['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + gitaly['enable'] = false + alertmanager['enable'] = false + node_exporter['enable'] = false + gitlab_rails['auto_migrate'] = false + ``` + 1. Run `sudo gitlab-ctl reconfigure`. 1. On `app1` apply the following changes to `/etc/gitlab/gitlab.rb`: - ```shell - gitlab_pages['enable'] = false - pages_external_url "http://<your-pages-domain>" - gitlab_rails['pages_path'] = "/mnt/pages" - ``` + ```shell + gitlab_pages['enable'] = false + pages_external_url "http://<your-pages-domain>" + gitlab_rails['pages_path'] = "/mnt/pages" + ``` 1. Run `sudo gitlab-ctl reconfigure`. diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 2100f7cd707..b2cad6cf926 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -102,50 +102,50 @@ The Pages daemon doesn't listen to the outside world. 1. Install the Pages daemon: - ``` - cd /home/git - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git - cd gitlab-pages - sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) - sudo -u git -H make - ``` + ``` + cd /home/git + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git + cd gitlab-pages + sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) + sudo -u git -H make + ``` 1. Go to the GitLab installation directory: - ```bash - cd /home/git/gitlab - ``` + ```bash + cd /home/git/gitlab + ``` 1. Edit `gitlab.yml` and under the `pages` setting, set `enabled` to `true` and the `host` to the FQDN under which GitLab Pages will be served: - ```yaml - ## GitLab Pages - pages: - enabled: true - # The location where pages are stored (default: shared/pages). - # path: shared/pages + ```yaml + ## GitLab Pages + pages: + enabled: true + # The location where pages are stored (default: shared/pages). + # path: shared/pages - host: example.io - port: 80 - https: false - ``` + host: example.io + port: 80 + https: false + ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in order to enable the pages daemon. In `gitlab_pages_options` the `-pages-domain` must match the `host` setting that you set above. - ``` - gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" - ``` + ``` + gitlab_pages_enabled=true + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" + ``` 1. Copy the `gitlab-pages` Nginx configuration file: - ```bash - sudo cp lib/support/nginx/gitlab-pages /etc/nginx/sites-available/gitlab-pages.conf - sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages.conf - ``` + ```bash + sudo cp lib/support/nginx/gitlab-pages /etc/nginx/sites-available/gitlab-pages.conf + sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages.conf + ``` 1. Restart NGINX 1. [Restart GitLab][restart] @@ -165,27 +165,27 @@ outside world. 1. Install the Pages daemon: - ``` - cd /home/git - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git - cd gitlab-pages - sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) - sudo -u git -H make - ``` + ``` + cd /home/git + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git + cd gitlab-pages + sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) + sudo -u git -H make + ``` 1. In `gitlab.yml`, set the port to `443` and https to `true`: - ```bash - ## GitLab Pages - pages: - enabled: true - # The location where pages are stored (default: shared/pages). - # path: shared/pages + ```bash + ## GitLab Pages + pages: + enabled: true + # The location where pages are stored (default: shared/pages). + # path: shared/pages - host: example.io - port: 443 - https: true - ``` + host: example.io + port: 443 + https: true + ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in order to enable the pages daemon. In `gitlab_pages_options` the @@ -193,17 +193,17 @@ outside world. The `-root-cert` and `-root-key` settings are the wildcard TLS certificates of the `example.io` domain: - ``` - gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key - ``` + ``` + gitlab_pages_enabled=true + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key + ``` 1. Copy the `gitlab-pages-ssl` Nginx configuration file: - ```bash - sudo cp lib/support/nginx/gitlab-pages-ssl /etc/nginx/sites-available/gitlab-pages-ssl.conf - sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages-ssl.conf - ``` + ```bash + sudo cp lib/support/nginx/gitlab-pages-ssl /etc/nginx/sites-available/gitlab-pages-ssl.conf + sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages-ssl.conf + ``` 1. Restart NGINX 1. [Restart GitLab][restart] @@ -231,48 +231,48 @@ world. Custom domains are supported, but no TLS. 1. Install the Pages daemon: - ``` - cd /home/git - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git - cd gitlab-pages - sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) - sudo -u git -H make - ``` + ``` + cd /home/git + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git + cd gitlab-pages + sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) + sudo -u git -H make + ``` 1. Edit `gitlab.yml` to look like the example below. You need to change the `host` to the FQDN under which GitLab Pages will be served. Set `external_http` to the secondary IP on which the pages daemon will listen for connections: - ```yaml - pages: - enabled: true - # The location where pages are stored (default: shared/pages). - # path: shared/pages + ```yaml + pages: + enabled: true + # The location where pages are stored (default: shared/pages). + # path: shared/pages - host: example.io - port: 80 - https: false + host: example.io + port: 80 + https: false - external_http: 192.0.2.2:80 - ``` + external_http: 192.0.2.2:80 + ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in order to enable the pages daemon. In `gitlab_pages_options` the `-pages-domain` and `-listen-http` must match the `host` and `external_http` settings that you set above respectively: - ``` - gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80" - ``` + ``` + gitlab_pages_enabled=true + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80" + ``` 1. Copy the `gitlab-pages-ssl` Nginx configuration file: - ```bash - sudo cp lib/support/nginx/gitlab-pages /etc/nginx/sites-available/gitlab-pages.conf - sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages.conf - ``` + ```bash + sudo cp lib/support/nginx/gitlab-pages /etc/nginx/sites-available/gitlab-pages.conf + sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages.conf + ``` 1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab @@ -297,33 +297,33 @@ world. Custom domains and TLS are supported. 1. Install the Pages daemon: - ``` - cd /home/git - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git - cd gitlab-pages - sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) - sudo -u git -H make - ``` + ``` + cd /home/git + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-pages.git + cd gitlab-pages + sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_PAGES_VERSION) + sudo -u git -H make + ``` 1. Edit `gitlab.yml` to look like the example below. You need to change the `host` to the FQDN under which GitLab Pages will be served. Set `external_http` and `external_https` to the secondary IP on which the pages daemon will listen for connections: - ```yaml - ## GitLab Pages - pages: - enabled: true - # The location where pages are stored (default: shared/pages). - # path: shared/pages + ```yaml + ## GitLab Pages + pages: + enabled: true + # The location where pages are stored (default: shared/pages). + # path: shared/pages - host: example.io - port: 443 - https: true + host: example.io + port: 443 + https: true - external_http: 192.0.2.2:80 - external_https: 192.0.2.2:443 - ``` + external_http: 192.0.2.2:80 + external_https: 192.0.2.2:443 + ``` 1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in order to enable the pages daemon. In `gitlab_pages_options` the @@ -332,17 +332,17 @@ world. Custom domains and TLS are supported. The `-root-cert` and `-root-key` settings are the wildcard TLS certificates of the `example.io` domain: - ``` - gitlab_pages_enabled=true - gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80 -listen-https 192.0.2.2:443 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key - ``` + ``` + gitlab_pages_enabled=true + gitlab_pages_options="-pages-domain example.io -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090 -listen-http 192.0.2.2:80 -listen-https 192.0.2.2:443 -root-cert /path/to/example.io.crt -root-key /path/to/example.io.key + ``` 1. Copy the `gitlab-pages-ssl` Nginx configuration file: - ```bash - sudo cp lib/support/nginx/gitlab-pages-ssl /etc/nginx/sites-available/gitlab-pages-ssl.conf - sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages-ssl.conf - ``` + ```bash + sudo cp lib/support/nginx/gitlab-pages-ssl /etc/nginx/sites-available/gitlab-pages-ssl.conf + sudo ln -sf /etc/nginx/sites-{available,enabled}/gitlab-pages-ssl.conf + ``` 1. Edit all GitLab related configs in `/etc/nginx/site-available/` and replace `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab @@ -359,9 +359,9 @@ are stored. If you wish to store them in another location you must set it up in `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['pages_path'] = "/mnt/storage/pages" - ``` + ```ruby + gitlab_rails['pages_path'] = "/mnt/storage/pages" + ``` 1. [Reconfigure GitLab][reconfigure] @@ -414,10 +414,10 @@ Pages access control is disabled by default. To enable it: 1. Modify your `config/gitlab.yml` file: - ```yaml - pages: - access_control: true - ``` + ```yaml + pages: + access_control: true + ``` 1. [Restart GitLab][restart]. 1. Create a new [system OAuth application](../../integration/oauth_provider.md#adding-an-application-through-the-profile). @@ -426,12 +426,12 @@ Pages access control is disabled by default. To enable it: application, but it does need the "api" scope. 1. Start the Pages daemon with the following additional arguments: - ```shell - -auth-client-secret <OAuth code generated by GitLab> \ - -auth-redirect-uri http://projects.example.io/auth \ - -auth-secret <40 random hex characters> \ - -auth-server <URL of the GitLab instance> - ``` + ```shell + -auth-client-secret <OAuth code generated by GitLab> \ + -auth-redirect-uri http://projects.example.io/auth \ + -auth-secret <40 random hex characters> \ + -auth-server <URL of the GitLab instance> + ``` 1. Users can now configure it in their [projects' settings](../../user/project/pages/introduction.md#gitlab-pages-access-control-core-only). @@ -444,12 +444,12 @@ are stored. If you wish to store them in another location you must set it up in `gitlab.yml` under the `pages` section: - ```yaml - pages: - enabled: true - # The location where pages are stored (default: shared/pages). - path: /mnt/storage/pages - ``` + ```yaml + pages: + enabled: true + # The location where pages are stored (default: shared/pages). + path: /mnt/storage/pages + ``` 1. [Restart GitLab][restart] diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index 78b2751da13..716a4259a64 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -65,8 +65,8 @@ To configure the pseudonymizer, you need to: ```yaml pseudonymizer: - manifest: config/pseudonymizer.yml - upload: + manifest: config/pseudonymizer.yml + upload: remote_directory: 'gitlab-elt' # bucket name connection: provider: AWS diff --git a/doc/administration/raketasks/geo.md b/doc/administration/raketasks/geo.md index 435aed8c413..691d34ab7fa 100644 --- a/doc/administration/raketasks/geo.md +++ b/doc/administration/raketasks/geo.md @@ -2,7 +2,7 @@ ## Git housekeeping -There are few tasks you can run to schedule a git housekeeping to start at the +There are few tasks you can run to schedule a git housekeeping to start at the next repository sync in a **Secondary node**: ### Incremental Repack @@ -23,7 +23,7 @@ sudo -u git -H bundle exec rake geo:git:housekeeping:incremental_repack RAILS_EN ### Full Repack -This is equivalent of running `git repack -d -A --pack-kept-objects` on a +This is equivalent of running `git repack -d -A --pack-kept-objects` on a _bare_ repository which will optionally, write a reachability bitmap index when this is enabled in GitLab. diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 0599e12b913..138db4dfbb1 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -2,14 +2,14 @@ >**Note:** > -> - [Introduced][ce-3050] in GitLab 8.9. -> - Importing will not be possible if the import instance version is lower -> than that of the exporter. -> - For existing installations, the project import option has to be enabled in -> application settings (`/admin/application_settings`) under 'Import sources'. -> - The exports are stored in a temporary [shared directory][tmp] and are deleted -> every 24 hours by a specific worker. -> - ImportExport can use object storage automatically starting from GitLab 11.3 +> - [Introduced][ce-3050] in GitLab 8.9. +> - Importing will not be possible if the import instance version is lower +> than that of the exporter. +> - For existing installations, the project import option has to be enabled in +> application settings (`/admin/application_settings`) under 'Import sources'. +> - The exports are stored in a temporary [shared directory][tmp] and are deleted +> every 24 hours by a specific worker. +> - ImportExport can use object storage automatically starting from GitLab 11.3 The GitLab Import/Export version can be checked by using: diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 42a1a1c2e60..2f83dd17d9f 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -1,17 +1,17 @@ # Repository Storage Rake Tasks -This is a collection of rake tasks you can use to help you list and migrate -existing projects and attachments associated with it from Legacy storage to +This is a collection of rake tasks you can use to help you list and migrate +existing projects and attachments associated with it from Legacy storage to the new Hashed storage type. You can read more about the storage types [here][storage-types]. ## Migrate existing projects to Hashed storage -Before migrating your existing projects, you should +Before migrating your existing projects, you should [enable hashed storage][storage-migration] for the new projects as well. -This task will schedule all your existing projects and attachments associated with it to be migrated to the +This task will schedule all your existing projects and attachments associated with it to be migrated to the **Hashed** storage type: **Omnibus Installation** @@ -30,15 +30,15 @@ They both also accept a range as environment variable: ```bash # to migrate any non migrated project from ID 20 to 50. -export ID_FROM=20 +export ID_FROM=20 export ID_TO=50 ``` You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. -There is a specific Queue you can watch to see how long it will take to finish: +There is a specific Queue you can watch to see how long it will take to finish: `hashed_storage:hashed_storage_project_migrate` -After it reaches zero, you can confirm every project has been migrated by running the commands bellow. +After it reaches zero, you can confirm every project has been migrated by running the commands bellow. If you find it necessary, you can run this migration script again to schedule missing projects. Any error or warning will be logged in Sidekiq's log file. @@ -55,7 +55,7 @@ If you need to rollback the storage migration for any reason, you can follow the NOTE: **Note:** Hashed Storage will be required in future version of GitLab. -To prevent new projects from being created in the Hashed storage, +To prevent new projects from being created in the Hashed storage, you need to undo the [enable hashed storage][storage-migration] changes. This task will schedule all your existing projects and associated attachments to be rolled back to the @@ -77,15 +77,14 @@ Both commands accept a range as environment variable: ```bash # to rollback any migrated project from ID 20 to 50. -export ID_FROM=20 +export ID_FROM=20 export ID_TO=50 ``` You can monitor the progress in the **Admin Area > Monitoring > Background Jobs** page. On the **Queues** tab, you can watch the `hashed_storage:hashed_storage_project_rollback` queue to see how long the process will take to finish. - -After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. +After it reaches zero, you can confirm every project has been rolled back by running the commands bellow. If some projects weren't rolled back, you can run this rollback script again to schedule further rollbacks. Any error or warning will be logged in Sidekiq's log file. @@ -106,7 +105,7 @@ sudo gitlab-rake gitlab:storage:legacy_projects sudo -u git -H bundle exec rake gitlab:storage:legacy_projects RAILS_ENV=production ``` ------- +--- To list projects using **Legacy** storage: @@ -139,7 +138,7 @@ sudo gitlab-rake gitlab:storage:hashed_projects sudo -u git -H bundle exec rake gitlab:storage:hashed_projects RAILS_ENV=production ``` ------- +--- To list projects using **Hashed** storage: @@ -171,7 +170,7 @@ sudo gitlab-rake gitlab:storage:legacy_attachments sudo -u git -H bundle exec rake gitlab:storage:legacy_attachments RAILS_ENV=production ``` ------- +--- To list project attachments using **Legacy** storage: @@ -203,7 +202,7 @@ sudo gitlab-rake gitlab:storage:hashed_attachments sudo -u git -H bundle exec rake gitlab:storage:hashed_attachments RAILS_ENV=production ``` ------- +--- To list project attachments using **Hashed** storage: diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index 54a423b9571..ae5ccfb9e37 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -4,16 +4,16 @@ You need `exiftool` installed on your system. If you installed GitLab: -- Using the Omnibus package, you're all set. -- From source, make sure `exiftool` is installed: +- Using the Omnibus package, you're all set. +- From source, make sure `exiftool` is installed: - ```sh - # Debian/Ubuntu - sudo apt-get install libimage-exiftool-perl + ```sh + # Debian/Ubuntu + sudo apt-get install libimage-exiftool-perl - # RHEL/CentOS - sudo yum install perl-Image-ExifTool - ``` + # RHEL/CentOS + sudo yum install perl-Image-ExifTool + ``` ## Remove EXIF data from existing uploads diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 8f7280d5128..098d946a9fa 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -10,43 +10,43 @@ an SMTP server, but you're not seeing mail delivered. Here's how to check the se 1. Run a Rails console: - ```sh - sudo gitlab-rails console production - ``` + ```sh + sudo gitlab-rails console production + ``` - or for source installs: + or for source installs: - ```sh - bundle exec rails console production - ``` + ```sh + bundle exec rails console production + ``` 1. Look at the ActionMailer `delivery_method` to make sure it matches what you intended. If you configured SMTP, it should say `:smtp`. If you're using Sendmail, it should say `:sendmail`: - ```ruby - irb(main):001:0> ActionMailer::Base.delivery_method - => :smtp - ``` + ```ruby + irb(main):001:0> ActionMailer::Base.delivery_method + => :smtp + ``` 1. If you're using SMTP, check the mail settings: - ```ruby - irb(main):002:0> ActionMailer::Base.smtp_settings - => {:address=>"localhost", :port=>25, :domain=>"localhost.localdomain", :user_name=>nil, :password=>nil, :authentication=>nil, :enable_starttls_auto=>true}``` - ``` + ```ruby + irb(main):002:0> ActionMailer::Base.smtp_settings + => {:address=>"localhost", :port=>25, :domain=>"localhost.localdomain", :user_name=>nil, :password=>nil, :authentication=>nil, :enable_starttls_auto=>true}``` + ``` - In the example above, the SMTP server is configured for the local machine. If this is intended, you may need to check your local mail - logs (e.g. `/var/log/mail.log`) for more details. + In the example above, the SMTP server is configured for the local machine. If this is intended, you may need to check your local mail + logs (e.g. `/var/log/mail.log`) for more details. -1. Send a test message via the console. +1. Send a test message via the console. - ```ruby - irb(main):003:0> Notify.test_email('youremail@email.com', 'Hello World', 'This is a test message').deliver_now - ``` + ```ruby + irb(main):003:0> Notify.test_email('youremail@email.com', 'Hello World', 'This is a test message').deliver_now + ``` - If you do not receive an e-mail and/or see an error message, then check - your mail server settings. + If you do not receive an e-mail and/or see an error message, then check + your mail server settings. ## Advanced Issues @@ -103,37 +103,37 @@ downtime. Otherwise skip to the next section. 1. Run `sudo gdb -p <PID>` to attach to the unicorn process. 1. In the gdb window, type: - ``` - call (void) rb_backtrace() - ``` + ``` + call (void) rb_backtrace() + ``` 1. This forces the process to generate a Ruby backtrace. Check `/var/log/gitlab/unicorn/unicorn_stderr.log` for the backtace. For example, you may see: - ```ruby - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:33:in `block in start' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:33:in `loop' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:36:in `block (2 levels) in start' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:44:in `sample' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `sample_objects' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `each_with_object' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `each' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:69:in `block in sample_objects' - from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:69:in `name' - ``` + ```ruby + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:33:in `block in start' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:33:in `loop' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:36:in `block (2 levels) in start' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:44:in `sample' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `sample_objects' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `each_with_object' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:68:in `each' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:69:in `block in sample_objects' + from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:69:in `name' + ``` 1. To see the current threads, run: - ``` - thread apply all bt - ``` + ``` + thread apply all bt + ``` 1. Once you're done debugging with `gdb`, be sure to detach from the process and exit: - ``` - detach - exit - ``` + ``` + detach + exit + ``` Note that if the unicorn process terminates before you are able to run these commands, gdb will report an error. To buy more time, you can always raise the @@ -162,21 +162,21 @@ separate Rails process to debug the issue: 1. Create a Personal Access Token for your user (Profile Settings -> Access Tokens). 1. Bring up the GitLab Rails console. For omnibus users, run: - ``` - sudo gitlab-rails console - ``` + ``` + sudo gitlab-rails console + ``` 1. At the Rails console, run: - ```ruby - [1] pry(main)> app.get '<URL FROM STEP 2>/?private_token=<TOKEN FROM STEP 3>' - ``` + ```ruby + [1] pry(main)> app.get '<URL FROM STEP 2>/?private_token=<TOKEN FROM STEP 3>' + ``` - For example: + For example: - ```ruby - [1] pry(main)> app.get 'https://gitlab.com/gitlab-org/gitlab-ce/issues/1?private_token=123456' - ``` + ```ruby + [1] pry(main)> app.get 'https://gitlab.com/gitlab-org/gitlab-ce/issues/1?private_token=123456' + ``` 1. In a new window, run `top`. It should show this ruby process using 100% CPU. Write down the PID. 1. Follow step 2 from the previous section on using gdb. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index c6529812ec3..277d42d06c6 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -1,11 +1,10 @@ # Uploads administration ->**Notes:** Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled. ## Using local storage ->**Notes:** +NOTE: **Note:** This is the default configuration To change the location where the uploads are stored locally, follow the steps @@ -15,7 +14,7 @@ below. **In Omnibus installations:** ->**Notes:** +NOTE: **Note:** For historical reasons, uploads are stored into a base directory, which by default is `uploads/-/system`. It is strongly discouraged to change this configuration option on an existing GitLab installation. _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/uploads`._ @@ -42,8 +41,8 @@ _The uploads are stored by default in ```yaml uploads: - storage_path: /mnt/storage - base_dir: uploads + storage_path: /mnt/storage + base_dir: uploads ``` 1. Save the file and [restart GitLab][] for the changes to take effect. @@ -108,8 +107,8 @@ _The uploads are stored by default in } ``` - >**Note:** - >If you are using AWS IAM profiles, be sure to omit the AWS access key and secret access key/value pairs. + NOTE: **Note:** + If you are using AWS IAM profiles, be sure to omit the AWS access key and secret access key/value pairs. ```ruby gitlab_rails['uploads_object_store_connection'] = { diff --git a/doc/api/README.md b/doc/api/README.md index 3ded230370c..b2bb55f2fcf 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -154,9 +154,10 @@ the `/Users` endpoint. The base URL is: `/api/scim/v2/groups/:group_path/Users/` ## Road to GraphQL -Going forward, we will start on moving to -[GraphQL](graphql/index.md) and deprecate the use of -controller-specific endpoints. GraphQL has a number of benefits: +[GraphQL](graphql/index.md) is available in GitLab, which will +allow deprecation of controller-specific endpoints. + +GraphQL has a number of benefits: 1. We avoid having to maintain two different APIs. 1. Callers of the API can request only what they need. diff --git a/doc/api/commits.md b/doc/api/commits.md index 25015fad9e3..6eb4c47415f 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -716,6 +716,7 @@ Example response if commit is signed: ``` Example response if commit is unsigned: + ```json { "message": "404 GPG Signature Not Found" diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index db073321808..bdc7c1959d2 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -1,6 +1,8 @@ # GraphQL API -> [Introduced][ce-19008] in GitLab 11.0. +> - [Introduced][ce-19008] in GitLab 11.0 (enabled by feature flag `graphql`). +> - [Always enabled](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30444) + in GitLab 12.1. [GraphQL](https://graphql.org/) is a query language for APIs that allows clients to request exactly the data they need, making it @@ -29,8 +31,6 @@ There are no plans to deprecate the REST API. To reduce the technical burden of supporting two APIs in parallel, they should share implementations as much as possible. -As of the 12.1 release, GraphQL is always enabled. - ## Available queries A first iteration of a GraphQL API includes the following queries @@ -47,6 +47,12 @@ info about multiplexed queries is also available for [graphql-ruby](https://graphql-ruby.org/queries/multiplex.html) the library GitLab uses on the backend. +## Reference + +GitLab's GraphQL reference [is available](reference/index.md). + +It is automatically generated from GitLab's GraphQL schema and embedded in a Markdown file. + ## GraphiQL The API can be explored by using the GraphiQL IDE, it is available on your diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md new file mode 100644 index 00000000000..2d3bec4ff67 --- /dev/null +++ b/doc/api/graphql/reference/index.md @@ -0,0 +1,507 @@ +<!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. +---> + +# GraphQL API Resources + +This documentation is self-generated based on GitLab current GraphQL schema. + +The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + +## Objects + +### AddAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | + +### AwardEmoji + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String! | The emoji name | +| `description` | String! | The emoji description | +| `unicode` | String! | The emoji in unicode | +| `emoji` | String! | The emoji as an icon | +| `unicodeVersion` | String! | The unicode version for this emoji | +| `user` | User! | The user who awarded the emoji | + +### Blob + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | +| `webUrl` | String | | +| `lfsOid` | String | | + +### Commit + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `sha` | String! | | +| `title` | String | | +| `description` | String | | +| `message` | String | | +| `authoredDate` | Time | | +| `webUrl` | String! | | +| `author` | User | | +| `latestPipeline` | Pipeline | Latest pipeline for this commit | + +### DetailedStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `group` | String! | | +| `icon` | String! | | +| `favicon` | String! | | +| `detailsPath` | String! | | +| `hasDetails` | Boolean! | | +| `label` | String! | | +| `text` | String! | | +| `tooltip` | String! | | + +### DiffPosition + +| Name | Type | Description | +| --- | ---- | ---------- | +| `headSha` | String! | The sha of the head at the time the comment was made | +| `baseSha` | String | The merge base of the branch the comment was made on | +| `startSha` | String! | The sha of the branch being compared against | +| `filePath` | String! | The path of the file that was changed | +| `oldPath` | String | The path of the file on the start sha. | +| `newPath` | String | The path of the file on the head sha. | +| `positionType` | DiffPositionType! | | +| `oldLine` | Int | The line on start sha that was changed | +| `newLine` | Int | The line on head sha that was changed | +| `x` | Int | The X postion on which the comment was made | +| `y` | Int | The Y position on which the comment was made | +| `width` | Int | The total width of the image | +| `height` | Int | The total height of the image | + +### Discussion + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `createdAt` | Time! | | + +### Group + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `path` | String! | | +| `fullName` | String! | | +| `fullPath` | ID! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `visibility` | String | | +| `lfsEnabled` | Boolean | | +| `requestAccessEnabled` | Boolean | | +| `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | +| `webUrl` | String! | | +| `avatarUrl` | String | | +| `parent` | Group | | + +### GroupPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readGroup` | Boolean! | Whether or not a user can perform `read_group` on this resource | + +### Issue + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | IssuePermissions! | Permissions for the current user on the resource | +| `iid` | ID! | | +| `title` | String! | | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `state` | IssueState! | | +| `reference` | String! | | +| `author` | User! | | +| `milestone` | Milestone | | +| `dueDate` | Time | | +| `confidential` | Boolean! | | +| `discussionLocked` | Boolean! | | +| `upvotes` | Int! | | +| `downvotes` | Int! | | +| `userNotesCount` | Int! | | +| `webPath` | String! | | +| `webUrl` | String! | | +| `relativePosition` | Int | | +| `closedAt` | Time | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `taskCompletionStatus` | TaskCompletionStatus! | | + +### IssuePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readIssue` | Boolean! | Whether or not a user can perform `read_issue` on this resource | +| `adminIssue` | Boolean! | Whether or not a user can perform `admin_issue` on this resource | +| `updateIssue` | Boolean! | Whether or not a user can perform `update_issue` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `reopenIssue` | Boolean! | Whether or not a user can perform `reopen_issue` on this resource | + +### Label + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `title` | String! | | +| `color` | String! | | +| `textColor` | String! | | + +### MergeRequest + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | MergeRequestPermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `iid` | String! | | +| `title` | String! | | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `state` | MergeRequestState! | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `sourceProject` | Project | | +| `targetProject` | Project! | | +| `project` | Project! | | +| `projectId` | Int! | | +| `sourceProjectId` | Int | | +| `targetProjectId` | Int! | | +| `sourceBranch` | String! | | +| `targetBranch` | String! | | +| `workInProgress` | Boolean! | | +| `mergeWhenPipelineSucceeds` | Boolean | | +| `diffHeadSha` | String | | +| `mergeCommitSha` | String | | +| `userNotesCount` | Int | | +| `shouldRemoveSourceBranch` | Boolean | | +| `forceRemoveSourceBranch` | Boolean | | +| `mergeStatus` | String | | +| `inProgressMergeCommitSha` | String | | +| `mergeError` | String | | +| `allowCollaboration` | Boolean | | +| `shouldBeRebased` | Boolean! | | +| `rebaseCommitSha` | String | | +| `rebaseInProgress` | Boolean! | | +| `mergeCommitMessage` | String | | +| `defaultMergeCommitMessage` | String | | +| `mergeOngoing` | Boolean! | | +| `sourceBranchExists` | Boolean! | | +| `mergeableDiscussionsState` | Boolean | | +| `webUrl` | String | | +| `upvotes` | Int! | | +| `downvotes` | Int! | | +| `subscribed` | Boolean! | | +| `headPipeline` | Pipeline | | +| `taskCompletionStatus` | TaskCompletionStatus! | | + +### MergeRequestPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readMergeRequest` | Boolean! | Whether or not a user can perform `read_merge_request` on this resource | +| `adminMergeRequest` | Boolean! | Whether or not a user can perform `admin_merge_request` on this resource | +| `updateMergeRequest` | Boolean! | Whether or not a user can perform `update_merge_request` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `pushToSourceBranch` | Boolean! | Whether or not a user can perform `push_to_source_branch` on this resource | +| `removeSourceBranch` | Boolean! | Whether or not a user can perform `remove_source_branch` on this resource | +| `cherryPickOnCurrentMergeRequest` | Boolean! | Whether or not a user can perform `cherry_pick_on_current_merge_request` on this resource | +| `revertOnCurrentMergeRequest` | Boolean! | Whether or not a user can perform `revert_on_current_merge_request` on this resource | + +### MergeRequestSetWipPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `mergeRequest` | MergeRequest | The merge request after mutation | + +### Metadata + +| Name | Type | Description | +| --- | ---- | ---------- | +| `version` | String! | | +| `revision` | String! | | + +### Milestone + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | | +| `title` | String! | | +| `state` | String! | | +| `dueDate` | Time | | +| `startDate` | Time | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | + +### Namespace + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `path` | String! | | +| `fullName` | String! | | +| `fullPath` | ID! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `visibility` | String | | +| `lfsEnabled` | Boolean | | +| `requestAccessEnabled` | Boolean | | + +### Note + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | NotePermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `project` | Project | The project this note is associated to | +| `author` | User! | The user who wrote this note | +| `resolvedBy` | User | The user that resolved the discussion | +| `system` | Boolean! | Whether or not this note was created by the system or by a user | +| `body` | String! | The content note itself | +| `bodyHtml` | String | The GitLab Flavored Markdown rendering of `note` | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `discussion` | Discussion | The discussion this note is a part of | +| `resolvable` | Boolean! | | +| `resolvedAt` | Time | The time the discussion was resolved | +| `position` | DiffPosition | The position of this note on a diff | + +### NotePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readNote` | Boolean! | Whether or not a user can perform `read_note` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `adminNote` | Boolean! | Whether or not a user can perform `admin_note` on this resource | +| `resolveNote` | Boolean! | Whether or not a user can perform `resolve_note` on this resource | +| `awardEmoji` | Boolean! | Whether or not a user can perform `award_emoji` on this resource | + +### PageInfo + +| Name | Type | Description | +| --- | ---- | ---------- | +| `hasNextPage` | Boolean! | When paginating forwards, are there more items? | +| `hasPreviousPage` | Boolean! | When paginating backwards, are there more items? | +| `startCursor` | String | When paginating backwards, the cursor to continue. | +| `endCursor` | String | When paginating forwards, the cursor to continue. | + +### Pipeline + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | PipelinePermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `iid` | String! | | +| `sha` | String! | | +| `beforeSha` | String | | +| `status` | PipelineStatusEnum! | | +| `detailedStatus` | DetailedStatus! | | +| `duration` | Int | Duration of the pipeline in seconds | +| `coverage` | Float | Coverage percentage | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `startedAt` | Time | | +| `finishedAt` | Time | | +| `committedAt` | Time | | + +### PipelinePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `updatePipeline` | Boolean! | Whether or not a user can perform `update_pipeline` on this resource | +| `adminPipeline` | Boolean! | Whether or not a user can perform `admin_pipeline` on this resource | +| `destroyPipeline` | Boolean! | Whether or not a user can perform `destroy_pipeline` on this resource | + +### Project + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | ProjectPermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `fullPath` | ID! | | +| `path` | String! | | +| `nameWithNamespace` | String! | | +| `name` | String! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `tagList` | String | | +| `sshUrlToRepo` | String | | +| `httpUrlToRepo` | String | | +| `webUrl` | String | | +| `starCount` | Int! | | +| `forksCount` | Int! | | +| `createdAt` | Time | | +| `lastActivityAt` | Time | | +| `archived` | Boolean | | +| `visibility` | String | | +| `containerRegistryEnabled` | Boolean | | +| `sharedRunnersEnabled` | Boolean | | +| `lfsEnabled` | Boolean | | +| `mergeRequestsFfOnlyEnabled` | Boolean | | +| `avatarUrl` | String | | +| `issuesEnabled` | Boolean | | +| `mergeRequestsEnabled` | Boolean | | +| `wikiEnabled` | Boolean | | +| `snippetsEnabled` | Boolean | | +| `jobsEnabled` | Boolean | | +| `publicJobs` | Boolean | | +| `openIssuesCount` | Int | | +| `importStatus` | String | | +| `onlyAllowMergeIfPipelineSucceeds` | Boolean | | +| `requestAccessEnabled` | Boolean | | +| `onlyAllowMergeIfAllDiscussionsAreResolved` | Boolean | | +| `printingMergeRequestLinkEnabled` | Boolean | | +| `namespace` | Namespace | | +| `group` | Group | | +| `statistics` | ProjectStatistics | | +| `repository` | Repository | | +| `mergeRequest` | MergeRequest | | +| `issue` | Issue | | + +### ProjectPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `changeNamespace` | Boolean! | Whether or not a user can perform `change_namespace` on this resource | +| `changeVisibilityLevel` | Boolean! | Whether or not a user can perform `change_visibility_level` on this resource | +| `renameProject` | Boolean! | Whether or not a user can perform `rename_project` on this resource | +| `removeProject` | Boolean! | Whether or not a user can perform `remove_project` on this resource | +| `archiveProject` | Boolean! | Whether or not a user can perform `archive_project` on this resource | +| `removeForkProject` | Boolean! | Whether or not a user can perform `remove_fork_project` on this resource | +| `removePages` | Boolean! | Whether or not a user can perform `remove_pages` on this resource | +| `readProject` | Boolean! | Whether or not a user can perform `read_project` on this resource | +| `createMergeRequestIn` | Boolean! | Whether or not a user can perform `create_merge_request_in` on this resource | +| `readWiki` | Boolean! | Whether or not a user can perform `read_wiki` on this resource | +| `readProjectMember` | Boolean! | Whether or not a user can perform `read_project_member` on this resource | +| `createIssue` | Boolean! | Whether or not a user can perform `create_issue` on this resource | +| `uploadFile` | Boolean! | Whether or not a user can perform `upload_file` on this resource | +| `readCycleAnalytics` | Boolean! | Whether or not a user can perform `read_cycle_analytics` on this resource | +| `downloadCode` | Boolean! | Whether or not a user can perform `download_code` on this resource | +| `downloadWikiCode` | Boolean! | Whether or not a user can perform `download_wiki_code` on this resource | +| `forkProject` | Boolean! | Whether or not a user can perform `fork_project` on this resource | +| `createProjectSnippet` | Boolean! | Whether or not a user can perform `create_project_snippet` on this resource | +| `readCommitStatus` | Boolean! | Whether or not a user can perform `read_commit_status` on this resource | +| `requestAccess` | Boolean! | Whether or not a user can perform `request_access` on this resource | +| `createPipeline` | Boolean! | Whether or not a user can perform `create_pipeline` on this resource | +| `createPipelineSchedule` | Boolean! | Whether or not a user can perform `create_pipeline_schedule` on this resource | +| `createMergeRequestFrom` | Boolean! | Whether or not a user can perform `create_merge_request_from` on this resource | +| `createWiki` | Boolean! | Whether or not a user can perform `create_wiki` on this resource | +| `pushCode` | Boolean! | Whether or not a user can perform `push_code` on this resource | +| `createDeployment` | Boolean! | Whether or not a user can perform `create_deployment` on this resource | +| `pushToDeleteProtectedBranch` | Boolean! | Whether or not a user can perform `push_to_delete_protected_branch` on this resource | +| `adminWiki` | Boolean! | Whether or not a user can perform `admin_wiki` on this resource | +| `adminProject` | Boolean! | Whether or not a user can perform `admin_project` on this resource | +| `updatePages` | Boolean! | Whether or not a user can perform `update_pages` on this resource | +| `adminRemoteMirror` | Boolean! | Whether or not a user can perform `admin_remote_mirror` on this resource | +| `createLabel` | Boolean! | Whether or not a user can perform `create_label` on this resource | +| `updateWiki` | Boolean! | Whether or not a user can perform `update_wiki` on this resource | +| `destroyWiki` | Boolean! | Whether or not a user can perform `destroy_wiki` on this resource | +| `createPages` | Boolean! | Whether or not a user can perform `create_pages` on this resource | +| `destroyPages` | Boolean! | Whether or not a user can perform `destroy_pages` on this resource | +| `readPagesContent` | Boolean! | Whether or not a user can perform `read_pages_content` on this resource | + +### ProjectStatistics + +| Name | Type | Description | +| --- | ---- | ---------- | +| `commitCount` | Int! | | +| `storageSize` | Int! | | +| `repositorySize` | Int! | | +| `lfsObjectsSize` | Int! | | +| `buildArtifactsSize` | Int! | | +| `packagesSize` | Int! | | +| `wikiSize` | Int | | + +### RemoveAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | + +### Repository + +| Name | Type | Description | +| --- | ---- | ---------- | +| `rootRef` | String | | +| `empty` | Boolean! | | +| `exists` | Boolean! | | +| `tree` | Tree | | + +### Submodule + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | + +### TaskCompletionStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `count` | Int! | | +| `completedCount` | Int! | | + +### ToggleAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | +| `toggledOn` | Boolean! | True when the emoji was awarded, false when it was removed | + +### Tree + +| Name | Type | Description | +| --- | ---- | ---------- | +| `lastCommit` | Commit | | + +### TreeEntry + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | +| `webUrl` | String | | + +### User + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String! | | +| `username` | String! | | +| `avatarUrl` | String! | | +| `webUrl` | String! | | + diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md new file mode 100644 index 00000000000..71a05b4d338 --- /dev/null +++ b/doc/api/group_clusters.md @@ -0,0 +1,280 @@ +# Group clusters API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30213) +in GitLab 12.1. + +NOTE: **Note:** +User will need at least maintainer access for the group to use these endpoints. + +## List group clusters + +Returns a list of group clusters. + +``` +GET /groups/:id/clusters +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | + +Example request: + +```bash +curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters +``` + +Example response: + +```json +[ + { + "id":18, + "name":"cluster-1", + "domain":"example.com", + "created_at":"2019-01-02T20:18:12.563Z", + "provider_type":"user", + "platform_type":"kubernetes", + "environment_scope":"*", + "cluster_type":"group_type", + "user": + { + "id":1, + "name":"Administrator", + "username":"root", + "state":"active", + "avatar_url":"https://www.gravatar.com/avatar/4249f4df72b..", + "web_url":"https://gitlab.example.com/root" + }, + "platform_kubernetes": + { + "api_url":"https://104.197.68.152", + "authorization_type":"rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----" + } + }, + { + "id":19, + "name":"cluster-2", + ... + } +] +``` + +## Get a single group cluster + +Gets a single group cluster. + +``` +GET /groups/:id/clusters/:cluster_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | + +Example request: + +```bash +curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/18 +``` + +Example response: + +```json +{ + "id":18, + "name":"cluster-1", + "domain":"example.com", + "created_at":"2019-01-02T20:18:12.563Z", + "provider_type":"user", + "platform_type":"kubernetes", + "environment_scope":"*", + "cluster_type":"group_type", + "user": + { + "id":1, + "name":"Administrator", + "username":"root", + "state":"active", + "avatar_url":"https://www.gravatar.com/avatar/4249f4df72b..", + "web_url":"https://gitlab.example.com/root" + }, + "platform_kubernetes": + { + "api_url":"https://104.197.68.152", + "authorization_type":"rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----" + }, + "group": + { + "id":26, + "name":"group-with-clusters-api", + "web_url":"https://gitlab.example.com/group-with-clusters-api" + } +} +``` + +## Add existing cluster to group + +Adds an existing Kubernetes cluster to the group. + +``` +POST /groups/:id/clusters/user +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `name` | String | yes | The name of the cluster | +| `domain` | String | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | +| `enabled` | Boolean | no | Determines if cluster is active or not, defaults to true | +| `managed` | Boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | +| `platform_kubernetes_attributes[api_url]` | String | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | String | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate | +| `platform_kubernetes_attributes[authorization_type]` | String | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | +| `environment_scope` | String | no | The associated environment to the cluster. Defaults to `*` **[PREMIUM]** | + +Example request: + +```bash +curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/user \ +-H "Accept: application/json" \ +-H "Content-Type:application/json" \ +--request POST --data '{"name":"cluster-5", "platform_kubernetes_attributes":{"api_url":"https://35.111.51.20","token":"12345","ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----"}}' +``` + +Example response: + +```json +{ + "id":24, + "name":"cluster-5", + "created_at":"2019-01-03T21:53:40.610Z", + "provider_type":"user", + "platform_type":"kubernetes", + "environment_scope":"*", + "cluster_type":"group_type", + "user": + { + "id":1, + "name":"Administrator", + "username":"root", + "state":"active", + "avatar_url":"https://www.gravatar.com/avatar/4249f4df72b..", + "web_url":"https://gitlab.example.com/root" + }, + "platform_kubernetes": + { + "api_url":"https://35.111.51.20", + "authorization_type":"rbac", + "ca_cert":"-----BEGIN CERTIFICATE-----\r\nhFiK1L61owwDQYJKoZIhvcNAQELBQAw\r\nLzEtMCsGA1UEAxMkZDA1YzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM4ZDBj\r\nMB4XDTE4MTIyNzIwMDM1MVoXDTIzMTIyNjIxMDM1MVowLzEtMCsGA1UEAxMkZDA1\r\nYzQ1YjctNzdiMS00NDY0LThjNmEtMTQ0ZDJkZjM.......-----END CERTIFICATE-----" + }, + "group": + { + "id":26, + "name":"group-with-clusters-api", + "web_url":"https://gitlab.example.com/root/group-with-clusters-api" + } +} +``` + +## Edit group cluster + +Updates an existing group cluster. + +``` +PUT /groups/:id/clusters/:cluster_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | +| `name` | String | no | The name of the cluster | +| `domain` | String | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | +| `platform_kubernetes_attributes[api_url]` | String | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | String | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate | +| `environment_scope` | String | no | The associated environment to the cluster **[PREMIUM]** | + +NOTE: **Note:** +`name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added +through the ["Add an existing Kubernetes Cluster"](../user/project/clusters/index.md#adding-an-existing-kubernetes-cluster) option or +through the ["Add existing cluster to group"](#add-existing-cluster-to-group) endpoint. + +Example request: + +```bash +curl --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/24 \ +-H "Content-Type:application/json" \ +--request PUT --data '{"name":"new-cluster-name","domain":"new-domain.com","api_url":"https://new-api-url.com"}' +``` + +Example response: + +```json +{ + "id":24, + "name":"new-cluster-name", + "domain":"new-domain.com", + "created_at":"2019-01-03T21:53:40.610Z", + "provider_type":"user", + "platform_type":"kubernetes", + "environment_scope":"*", + "cluster_type":"group_type", + "user": + { + "id":1, + "name":"Administrator", + "username":"root", + "state":"active", + "avatar_url":"https://www.gravatar.com/avatar/4249f4df72b..", + "web_url":"https://gitlab.example.com/root" + }, + "platform_kubernetes": + { + "api_url":"https://new-api-url.com", + "authorization_type":"rbac", + "ca_cert":null + }, + "group": + { + "id":26, + "name":"group-with-clusters-api", + "web_url":"https://gitlab.example.com/group-with-clusters-api" + } +} + +``` + +## Delete group cluster + +Deletes an existing group cluster. + +``` +DELETE /groups/:id/clusters/:cluster_id +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | + +Example request: + +```bash +curl --request DELETE --header 'Private-Token: <your_access_token>' https://gitlab.example.com/api/v4/groups/26/clusters/23' +``` diff --git a/doc/api/groups.md b/doc/api/groups.md index d05e4b29fef..0d500f783aa 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -634,11 +634,13 @@ Parameters: By default, groups only get 20 namespaces at a time because the API results are paginated. To get more (up to 100), pass the following as an argument to the API call: + ``` /groups?per_page=100 ``` And to switch pages add: + ``` /groups?per_page=100&page=2 ``` diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index de87e4a0aee..662a4b3e424 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -1331,9 +1331,11 @@ If you don't have permissions to accept this merge request - you'll get a `401` If the merge request is already merged or closed - you get `405` and error message 'Method Not Allowed' In case the merge request is not set to be merged when the pipeline succeeds, you'll also get a `406` error. + ``` PUT /projects/:id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds ``` + Parameters: - `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md index 3adca61a108..fb6fa040244 100644 --- a/doc/api/protected_tags.md +++ b/doc/api/protected_tags.md @@ -5,6 +5,7 @@ **Valid access levels** Currently, these levels are recognized: + ``` 0 => No access 30 => Developer access diff --git a/doc/api/repositories.md b/doc/api/repositories.md index ffae5c17310..4aff79c9c62 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -235,17 +235,17 @@ Example response: ```json { - "id": "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863", - "short_id": "1a0b36b3", - "title": "Initial commit", - "created_at": "2014-02-27T08:03:18.000Z", - "parent_ids": [], - "message": "Initial commit\n", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "authored_date": "2014-02-27T08:03:18.000Z", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T08:03:18.000Z" + "id": "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863", + "short_id": "1a0b36b3", + "title": "Initial commit", + "created_at": "2014-02-27T08:03:18.000Z", + "parent_ids": [], + "message": "Initial commit\n", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "authored_date": "2014-02-27T08:03:18.000Z", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T08:03:18.000Z" } ``` diff --git a/doc/api/repository_submodules.md b/doc/api/repository_submodules.md index 2c44c4abc93..5a722a75cb9 100644 --- a/doc/api/repository_submodules.md +++ b/doc/api/repository_submodules.md @@ -43,7 +43,7 @@ Example response: "ae1d9fb46aa2b07ee9836d49862ec4e2c46fbbba" ], "committed_date": "2018-09-20T09:26:24.000-07:00", - "authored_date": "2018-09-20T09:26:24.000-07:00", + "authored_date": "2018-09-20T09:26:24.000-07:00", "status": null } ``` diff --git a/doc/api/tags.md b/doc/api/tags.md index 3177fec618f..af86ba961f4 100644 --- a/doc/api/tags.md +++ b/doc/api/tags.md @@ -141,6 +141,7 @@ Parameters: "message": null } ``` + The message will be `null` when creating a lightweight tag otherwise it will contain the annotation. diff --git a/doc/api/users.md b/doc/api/users.md index 213d1865aca..b43b4de823b 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -595,7 +595,7 @@ Example responses ## User counts -Get the counts (same as in top right menu) of the currently signed in user. +Get the counts (same as in top right menu) of the currently signed in user. | Attribute | Type | Description | | --------- | ---- | ----------- | diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index cc6e5d3960b..eaa4c13de55 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -30,6 +30,7 @@ GET /projects/:id/vulnerabilities?scope=all GET /projects/:id/vulnerabilities?scope=dismissed GET /projects/:id/vulnerabilities?severity=high GET /projects/:id/vulnerabilities?confidence=unknown,experimental +GET /projects/:id/vulnerabilities?pipeline_id=42 ``` | Attribute | Type | Required | Description | @@ -39,6 +40,7 @@ GET /projects/:id/vulnerabilities?confidence=unknown,experimental | `scope` | string | no | Returns vulnerabilities for the given scope: `all` or `dismissed`. Defaults to `dismissed` | | `severity` | string array | no | Returns vulnerabilities belonging to specified severity level: `undefined`, `info`, `unknown`, `low`, `medium`, `high`, or `critical`. Defaults to all' | | `confidence` | string array | no | Returns vulnerabilities belonging to specified confidence level: `undefined`, `ignore`, `unknown`, `experimental`, `low`, `medium`, `high`, or `confirmed`. Defaults to all | +| `pipeline_id` | integer/string | no | Returns vulnerabilities belonging to specified pipeline. | ```bash curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/4/vulnerabilities diff --git a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md index 538843ab8dc..1d4c9221cf2 100644 --- a/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md +++ b/doc/ci/examples/deploy_spring_boot_to_cloud_foundry/index.md @@ -79,10 +79,10 @@ image: java:8 stages: - build - deploy - + before_script: - chmod +x mvnw - + build: stage: build script: ./mvnw package diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index d7308a3a5ec..1576efd5a7d 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -452,6 +452,7 @@ To start using Container Registry on our machine, we first need to login to the ```bash docker login registry.gitlab.com ``` + Then we can build and push our image to GitLab: ```bash diff --git a/doc/ci/examples/php.md b/doc/ci/examples/php.md index 0b9e9e93e55..1dd3049d53d 100644 --- a/doc/ci/examples/php.md +++ b/doc/ci/examples/php.md @@ -6,7 +6,7 @@ type: tutorial This guide covers basic building instructions for PHP projects. -Two testing scenarios are covered: using the Docker executor and +Two testing scenarios are covered: using the Docker executor and using the Shell executor. ## Test PHP projects using the Docker executor diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 1354a26d6e2..cce33c7a6b4 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -69,12 +69,14 @@ correctly with your CI jobs: 1. Next, if you are using `gitlab-runner` v1.10+, you can set the `GIT_SUBMODULE_STRATEGY` variable to either `normal` or `recursive` to tell the runner to fetch your submodules before the job: - ```yaml - variables: - GIT_SUBMODULE_STRATEGY: recursive - ``` - See the [`.gitlab-ci.yml` reference](yaml/README.md#git-submodule-strategy) - for more details about `GIT_SUBMODULE_STRATEGY`. + + ```yaml + variables: + GIT_SUBMODULE_STRATEGY: recursive + ``` + + See the [`.gitlab-ci.yml` reference](yaml/README.md#git-submodule-strategy) + for more details about `GIT_SUBMODULE_STRATEGY`. 1. If you are using an older version of `gitlab-runner`, then use `git submodule sync/update` in `before_script`: diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md index 6c73cbbf8d7..58307660e51 100644 --- a/doc/ci/interactive_web_terminal/index.md +++ b/doc/ci/interactive_web_terminal/index.md @@ -8,7 +8,7 @@ type: reference Interactive web terminals give the user access to a terminal in GitLab for running one-off commands for their CI pipeline. Since this is giving the user -shell access to the environment where [GitLab Runner](https://docs.gitlab.com/runner/) +shell access to the environment where [GitLab Runner](https://docs.gitlab.com/runner/) is deployed, some [security precautions](../../administration/integration/terminal.md#security) were taken to protect the users. @@ -62,4 +62,3 @@ close the terminal window. ## Interactive Web Terminals for the Web IDE **(ULTIMATE ONLY)** Read the Web IDE docs to learn how to run [Interactive Terminals through the Web IDE](../../user/project/web_ide/index.md). - diff --git a/doc/ci/services/postgres.md b/doc/ci/services/postgres.md index 960346ac11a..b72dd6e920a 100644 --- a/doc/ci/services/postgres.md +++ b/doc/ci/services/postgres.md @@ -70,7 +70,7 @@ template1=# CREATE USER runner WITH PASSWORD '$password' CREATEDB; ``` *__Note:__ Notice that we created the user with the privilege to be able to -create databases (`CREATEDB`). In the following steps we will create a database +create databases (`CREATEDB`). In the following steps we will create a database explicitly for that user but having that privilege can be useful if in your testing framework you have tools that drop and create databases.* diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 42dd4f08ed8..4d6ca8cff6d 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -273,6 +273,7 @@ export CI_RUNNER_ID="10" export CI_RUNNER_DESCRIPTION="my runner" export CI_RUNNER_TAGS="docker, linux" export CI_SERVER="yes" +export CI_SERVER_HOST="example.com" export CI_SERVER_NAME="GitLab" export CI_SERVER_REVISION="70606bf" export CI_SERVER_VERSION="8.9.0" @@ -644,6 +645,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ CI_PROJECT_DIR=/builds/gitlab-examples/ci-debug-trace ++ export CI_SERVER=yes ++ CI_SERVER=yes +++ export 'CI_SERVER_HOST=example.com' +++ CI_SERVER_HOST='example.com' ++ export 'CI_SERVER_NAME=GitLab CI' ++ CI_SERVER_NAME='GitLab CI' ++ export CI_SERVER_VERSION= @@ -678,6 +681,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ CI_JOB_NAME=debug_trace ++ export CI_JOB_STAGE=test ++ CI_JOB_STAGE=test +++ export CI_SERVER_HOST=example.com +++ CI_SERVER_HOST=example.com ++ export CI_SERVER_NAME=GitLab ++ CI_SERVER_NAME=GitLab ++ export CI_SERVER_VERSION=8.14.3-ee diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index e911e97d3c8..49543c57886 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -101,6 +101,7 @@ future GitLab releases.** | `CI_RUNNER_TAGS` | 8.10 | 0.5 | The defined runner tags | | `CI_RUNNER_VERSION` | all | 10.6 | GitLab Runner version that is executing the current job | | `CI_SERVER` | all | all | Mark that job is executed in CI environment | +| `CI_SERVER_HOST` | 12.1 | all | Host component of the GitLab instance URL, without protocol and port (like gitlab.example.com) | | `CI_SERVER_NAME` | all | all | The name of CI server that is used to coordinate jobs | | `CI_SERVER_REVISION` | all | all | GitLab revision that is used to schedule jobs | | `CI_SERVER_VERSION` | all | all | GitLab version that is used to schedule jobs | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 474db05de06..c2ef58acf15 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2829,7 +2829,8 @@ Alternatively, one can pass the `ci.skip` [Git push option][push-option] if using Git 2.10 or newer: ```sh -git push -o ci.skip +git push --push-option=ci.skip # using git 2.10+ +git push -o ci.skip # using git 2.18+ ``` <!-- ## Troubleshooting diff --git a/doc/development/README.md b/doc/development/README.md index 1566173992a..a74770ae383 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -22,7 +22,7 @@ description: 'Learn how to contribute to GitLab.' - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) - [Requesting access to Chatops on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLabbers) -## UX and frontend guides +## UX and Frontend guides - [GitLab Design System](https://design.gitlab.com/) for building GitLab with existing CSS styles and elements - [Frontend guidelines](fe_guide/index.md) diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index c83a0427c98..7569ccc04c1 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -424,12 +424,8 @@ Will generate a field called `mergeRequestSetWip` that ### Authorizing resources -To authorize resources inside a mutation, we can include the -`Gitlab::Graphql::Authorize::AuthorizeResource` concern in the -mutation. - -This allows us to provide the required abilities on the mutation like -this: +To authorize resources inside a mutation, we first provide the required + abilities on the mutation like this: ```ruby module Mutations diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 3ed586f07e9..bd07a01e782 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -129,6 +129,7 @@ merge_request: author: type: ``` + If you're working on the GitLab EE repository, the entry will be added to `ee/changelogs/unreleased/` instead. diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 403a5b21827..b3406275937 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -15,23 +15,19 @@ Currently, there are four endpoints for simulating the following conditions: ## Enabling chaos endpoints -For obvious reasons, these endpoints are not enabled by default. They can be enabled by setting the `GITLAB_ENABLE_CHAOS_ENDPOINTS` environment variable to `1`. - -For example, if you're using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: - -```bash -GITLAB_ENABLE_CHAOS_ENDPOINTS=1 gdk run -``` - -## Securing the chaos endpoints +For obvious reasons, these endpoints are not enabled by default on `production`. +They are enabled by default on **development** environments. DANGER: **Danger:** -It is highly recommended that you secure access to the chaos endpoints using a secret token. This is recommended when enabling these endpoints locally and essential when running in a staging or other shared environment. You should not enable them in production unless you absolutely know what you're doing. +It is required that you secure access to the chaos endpoints using a secret token. +You should not enable them in production unless you absolutely know what you're doing. -A secret token can be set through the `GITLAB_CHAOS_SECRET` environment variable. For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: +A secret token can be set through the `GITLAB_CHAOS_SECRET` environment variable. +For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) +this can be done with the following command: ```bash -GITLAB_ENABLE_CHAOS_ENDPOINTS=1 GITLAB_CHAOS_SECRET=secret gdk run +GITLAB_CHAOS_SECRET=secret gdk run ``` Replace `secret` with your own secret token. @@ -56,10 +52,11 @@ GET /-/chaos/leakmem?memory_mb=1024&duration_s=50 | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ---------------------------------------------------------------------------------- | | `memory_mb` | integer | no | How much memory, in MB, should be leaked. Defaults to 100MB. | -| `duration_s` | integer | no | Minimum duration, in seconds, that the memory should be retained. Defaults to 30s. | +| `duration_s` | integer | no | Minimum duration_s, in seconds, that the memory should be retained. Defaults to 30s. | ```bash curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10&token=secret ``` ## CPU spin @@ -70,23 +67,47 @@ Depending on your rack server setup, your request may timeout after a predermine If you're using Unicorn, this is done by killing the worker process. ``` -GET /-/chaos/cpuspin -GET /-/chaos/cpuspin?duration_s=50 +GET /-/chaos/cpu_spin +GET /-/chaos/cpu_spin?duration_s=50 +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------------------------------------------------------- | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | + +```bash +curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60&token=secret +``` + +## DB spin + +This endpoint attempts to fully utilise a single core, and interleave it with DB request, for the given period. +This endpoint can be used to model yielding execution to another threads when running concurrently. + +Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). +If you're using Unicorn, this is done by killing the worker process. + +``` +GET /-/chaos/db_spin +GET /-/chaos/db_spin?duration_s=50 ``` | Attribute | Type | Required | Description | | ------------ | ------- | -------- | --------------------------------------------------------------------- | +| `interval_s` | float | no | Interval, in seconds, for every DB request. Defaults to 1s | | `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | ```bash -curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/db_spin?interval_s=1&duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/db_spin?interval_s=1&duration_s=60&token=secret ``` ## Sleep -This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such as network calls to backend services. It will sleep for a given duration. +This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such as network calls to backend services. It will sleep for a given duration_s. -As with the CPU Spin endpoint, this may lead to your request timing out if duration exceeds the configured limit. +As with the CPU Spin endpoint, this may lead to your request timing out if duration_s exceeds the configured limit. ``` GET /-/chaos/sleep @@ -99,6 +120,7 @@ GET /-/chaos/sleep?duration_s=50 ```bash curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/sleep?duration_s=60&token=secret ``` ## Kill @@ -114,4 +136,5 @@ GET /-/chaos/kill ```bash curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/kill?token=secret ``` diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index 910f9f4bf7a..d0562cd2bbd 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -58,9 +58,9 @@ issue is labeled with a subject label corresponding to your expertise. Subject labels are always all-lowercase. -## Team labels +## Team labels -**Important**: Most of the team labels will be soon deprecated in favor of [Group labels](#group-labels). +**Important**: Most of the team labels will be soon deprecated in favor of [Group labels](#group-labels). Team labels specify what team is responsible for this issue. Assigning a team label makes sure issues get the attention of the appropriate @@ -98,7 +98,6 @@ indicate if an issue needs backend work, frontend work, or both. Team labels are always capitalized so that they show up as the first label for any issue. - ## Stage labels Stage labels specify which [DevOps stage][devops-stages] the issue belongs to. @@ -141,43 +140,42 @@ Group labels specify which [groups][structure-groups] the issue belongs to. The current group labels are: -* ~"group::access" -* ~"group::measure" -* ~"group::source code" -* ~"group::knowledge" -* ~"group::editor" -* ~"group::gitaly" -* ~"group::gitter" -* ~"group::team planning" -* ~"group::enterprise planning" -* ~"group::certify" -* ~"group::ci and runner" -* ~"group::testing" -* ~"group::package" -* ~"group::progressive delivery" -* ~"group::release management" -* ~"group::autodevops and kubernetes" -* ~"group::serverless and paas" -* ~"group::apm" -* ~"group::health" -* ~"group::static analysis" -* ~"group::dynamic analysis" -* ~"group::software composition analysis" -* ~"group::runtime application security" -* ~"group::threat management" -* ~"group::application infrastructure security" -* ~"group::activation" -* ~"group::adoption" -* ~"group::upsell" -* ~"group::retention" -* ~"group::fulfillment" -* ~"group::telemetry" -* ~"group::distribution" -* ~"group::geo" -* ~"group::memory" -* ~"group::ecosystem" +- ~"group::access" +- ~"group::measure" +- ~"group::source code" +- ~"group::knowledge" +- ~"group::editor" +- ~"group::gitaly" +- ~"group::gitter" +- ~"group::team planning" +- ~"group::enterprise planning" +- ~"group::certify" +- ~"group::ci and runner" +- ~"group::testing" +- ~"group::package" +- ~"group::progressive delivery" +- ~"group::release management" +- ~"group::autodevops and kubernetes" +- ~"group::serverless and paas" +- ~"group::apm" +- ~"group::health" +- ~"group::static analysis" +- ~"group::dynamic analysis" +- ~"group::software composition analysis" +- ~"group::runtime application security" +- ~"group::threat management" +- ~"group::application infrastructure security" +- ~"group::activation" +- ~"group::adoption" +- ~"group::upsell" +- ~"group::retention" +- ~"group::fulfillment" +- ~"group::telemetry" +- ~"group::distribution" +- ~"group::geo" +- ~"group::memory" +- ~"group::ecosystem" - These labels are [scoped labels](../../user/project/labels.md#scoped-labels-premium) and thus are mutually exclusive. @@ -192,15 +190,15 @@ can be applied to a single issue. You can find the groups listed in the The current department labels are: -* ~UX -* ~Quality +- ~UX +- ~Quality ## Specialization labels These labels narrow the [specialization](https://about.gitlab.com/company/team/structure/#specialist) on a unit of work. -* ~frontend -* ~backend +- ~frontend +- ~backend ## Release Scoping labels @@ -248,9 +246,9 @@ There can be multiple facets of the impact. The below is a guideline. If a bug seems to fall between two severity labels, assign it to the higher-severity label. - Example(s) of ~S1 - - Data corruption/loss. + - Data corruption/loss. - Security breach. - - Unable to create an issue or merge request. + - Unable to create an issue or merge request. - Unable to add a comment or discussion to the issue or merge request. - Example(s) of ~S2 - Cannot submit changes through the web IDE but the commandline works. diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index cbdc0a3a174..36a2f47a55b 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -43,7 +43,7 @@ Meanwhile, anyone can contribute [documentation improvements](improvement-workfl ## Markdown and styles -[GitLab docs](https://gitlab.com/gitlab-com/gitlab-docs) uses [GitLab Kramdown](https://gitlab.com/gitlab-org/gitlab_kramdown) +[GitLab docs](https://gitlab.com/gitlab-org/gitlab-docs) uses [GitLab Kramdown](https://gitlab.com/gitlab-org/gitlab_kramdown) as its markdown rendering engine. See the [GitLab Markdown Guide](https://about.gitlab.com/handbook/product/technical-writing/markdown-guide/) for a complete Kramdown reference. Adhere to the [Documentation Style Guide](styleguide.md). If a style standard is missing, you are welcome to suggest one via a merge request. @@ -384,7 +384,7 @@ on how the left-side navigation menu is built and updated. NOTE: **Note:** To preview your changes to documentation locally, follow this -[development guide](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/README.md#development-when-contributing-to-gitlab-documentation) or [these instructions for GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/gitlab_docs.md). +[development guide](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/README.md#development-when-contributing-to-gitlab-documentation) or [these instructions for GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/gitlab_docs.md). The live preview is currently enabled for the following projects: @@ -408,7 +408,7 @@ You will need to push a branch to those repositories, it doesn't work for forks. The `review-docs-deploy*` job will: -1. Create a new branch in the [gitlab-docs](https://gitlab.com/gitlab-com/gitlab-docs) +1. Create a new branch in the [gitlab-docs](https://gitlab.com/gitlab-org/gitlab-docs) project named after the scheme: `$DOCS_GITLAB_REPO_SUFFIX-$CI_ENVIRONMENT_SLUG`, where `DOCS_GITLAB_REPO_SUFFIX` is the suffix for each product, e.g, `ce` for CE, etc. @@ -464,7 +464,7 @@ If you want to know the in-depth details, here's what's really happening: 1. The preview URL is shown both at the job output and in the merge request widget. You also get the link to the remote pipeline. 1. In the docs project, the pipeline is created and it - [skips the test jobs](https://gitlab.com/gitlab-com/gitlab-docs/blob/8d5d5c750c602a835614b02f9db42ead1c4b2f5e/.gitlab-ci.yml#L50-55) + [skips the test jobs](https://gitlab.com/gitlab-org/gitlab-docs/blob/8d5d5c750c602a835614b02f9db42ead1c4b2f5e/.gitlab-ci.yml#L50-55) to lower the build time. 1. Once the docs site is built, the HTML files are uploaded as artifacts. 1. A specific Runner tied only to the docs project, runs the Review App job @@ -488,7 +488,7 @@ Currently, the following tests are in place: that all cURL examples in API docs use the full switches. It's recommended to [check locally](#previewing-the-changes-live) before pushing to GitLab by executing the command `bundle exec nanoc check internal_links` on your local - [`gitlab-docs`](https://gitlab.com/gitlab-com/gitlab-docs) directory. + [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. 1. [`ee_compat_check`](../automatic_ce_ee_merge.md#avoiding-ce-ee-merge-conflicts-beforehand) (runs on CE only): When you submit a merge request to GitLab Community Edition (CE), there is this additional job that runs against Enterprise Edition (EE) diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 20eeebf444f..753a636a779 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -4,9 +4,9 @@ description: "Learn how GitLab docs' global navigation works and how to add new # Global navigation -> - [Introduced](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/362) +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/362) in GitLab 11.6. -> - [Updated](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/482) in GitLab 12.1. +> - [Updated](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/482) in GitLab 12.1. The global nav adds to the left sidebar the ability to navigate and explore the contents of GitLab's documentation. @@ -25,7 +25,7 @@ To add a new doc to the nav, first and foremost, check with the technical writin Once you get their approval and their guidance in regards to the position on the nav, read trhough this page to understand how it works, and submit a merge request to the docs site, adding the doc you wish to include in the nav into the -[global nav data file](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/content/_data/global-nav.yaml). +[global nav data file](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/global-nav.yaml). Don't forget to ask a technical writer to review your changes before merging. @@ -70,7 +70,7 @@ the data among the nav in containers properly [styled](#css-classes). ### Data file -The [data file](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/content/_data/global-nav.yaml) +The [data file](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/content/_data/global-nav.yaml) is structured in three components: sections, categories, and docs. #### Sections @@ -248,9 +248,9 @@ Examples: ### Layout file (logic) -The [layout](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/layouts/global_nav.html) +The [layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/layouts/global_nav.html) is fed by the [data file](#data-file), builds the global nav, and is rendered by the -[default](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/layouts/default.html) layout. +[default](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/layouts/default.html) layout. There are three main considerations on the logic built for the nav: diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 6dd12b5efa7..1aef0ed855c 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -4,14 +4,14 @@ description: "Learn how GitLab's documentation website is architectured." # Documentation site architecture -Learn how we build and architecture [`gitlab-docs`](https://gitlab.com/gitlab-com/gitlab-docs) +Learn how we build and architecture [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) and deploy it to <https://docs.gitlab.com>. ## Repository While the source of the documentation content is stored in GitLab's respective product repositories, the source that is used to build the documentation site _from that content_ -is located at <https://gitlab.com/gitlab-com/gitlab-docs>. +is located at <https://gitlab.com/gitlab-org/gitlab-docs>. The following diagram illustrates the relationship between the repositories from where content is sourced, the `gitlab-docs` project, and the published output. @@ -43,7 +43,7 @@ from where content is sourced, the `gitlab-docs` project, and the published outp G --> L ``` -See the [README there](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/README.md) +See the [README there](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/README.md) for detailed information. ## Assets @@ -76,7 +76,7 @@ read through the [global navigation](global_nav.md) doc. The docs site is deployed to production with GitLab Pages, and previewed in merge requests with Review Apps. -The deployment aspects will be soon transferred from the [original document](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/README.md) +The deployment aspects will be soon transferred from the [original document](https://gitlab.com/gitlab-org/gitlab-docs/blob/master/README.md) to this page. <!-- diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index fe676efa94d..025a946da0e 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -127,7 +127,7 @@ Notes: ## Help and feedback section -The "help and feedback" section (introduced by [!319](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/319)) displayed at the end of each document +The "help and feedback" section (introduced by [!319](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/319)) displayed at the end of each document can be omitted from the doc by adding a key into the its frontmatter: ```yaml @@ -142,7 +142,7 @@ you must check with a technical writer before doing so. ### Disqus We also have integrated the docs site with Disqus (introduced by -[!151](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/151)), +[!151](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/151)), allowing our users to post comments. To omit only the comments from the feedback section, use the following diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index d9cea0614c3..1ca965f8ae9 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -340,7 +340,7 @@ For other punctuation rules, please refer to the links shift too, which eventually leads to dead links. If you think it is compelling to add numbers in headings, make sure to at least discuss it with someone in the Merge Request. -- [Avoid using symbols and special chars](https://gitlab.com/gitlab-com/gitlab-docs/issues/84) +- [Avoid using symbols and special chars](https://gitlab.com/gitlab-org/gitlab-docs/issues/84) in headers. Whenever possible, they should be plain and short text. - Avoid adding things that show ephemeral statuses. For example, if a feature is considered beta or experimental, put this info in a note, not in the heading. @@ -488,7 +488,7 @@ You can link any up-to-date video that is useful to the GitLab user. ### Embed videos -> [Introduced](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/472) in GitLab 12.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/472) in GitLab 12.1. GitLab docs (docs.gitlab.com) support embedded videos. @@ -801,7 +801,7 @@ GitLab.com Free, and all higher tiers. ### How it works -Introduced by [!244](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/244), +Introduced by [!244](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/244), the special markup `**(STARTER)**` will generate a `span` element to trigger the badges and tooltips (`<span class="badge-trigger starter">`). When the keyword "only" is added, the corresponding GitLab.com badge will not be displayed. diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 0abfe4b82a4..9f488fac7d0 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -6,5 +6,5 @@ description: Learn the processes for contributing to GitLab's documentation. Documentation workflows at GitLab differ depending on the reason for the change: -- [Documentation process for feature changes](feature-change-workflow.md) - The documentation is being created or updated as part of the development and release of a new or enhanced feature. This process involves the developer of the feature (who includes new/updated documentation files as part of the same merge request containing the feature's code) and also involves the product manager and technical writer who are listed for the feature's [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). +- [Documentation process for feature changes](feature-change-workflow.md) - The documentation is being created or updated as part of the development and release of a new or enhanced feature. This process involves the developer of the feature (who includes new/updated documentation files as part of the same merge request containing the feature's code) and also involves the product manager and technical writer who are listed for the feature's [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). - [Documentation improvement workflow](improvement-workflow.md) - All documentation additions not associated with a feature release. Documentation is being created or updated to improve accuracy, completeness, ease of use, or any reason other than a feature change. Anyone (and everyone) can contribute a merge request for this type of change at any time. diff --git a/doc/development/fe_guide/design_patterns.md b/doc/development/fe_guide/design_patterns.md index 0342d16a87c..2f372f783f5 100644 --- a/doc/development/fe_guide/design_patterns.md +++ b/doc/development/fe_guide/design_patterns.md @@ -53,6 +53,7 @@ When writing a class that needs to manipulate the DOM guarantee a container opti This is useful when we need that class to be instantiated more than once in the same page. Bad: + ```javascript class Foo { constructor() { @@ -63,6 +64,7 @@ new Foo(); ``` Good: + ```javascript class Foo { constructor(opts) { @@ -72,6 +74,7 @@ class Foo { new Foo({ container: '.my-element' }); ``` + You can find an example of the above in this [class][container-class-example]; [container-class-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/app/assets/javascripts/mini_pipeline_graph_dropdown.js diff --git a/doc/development/fe_guide/droplab/droplab.md b/doc/development/fe_guide/droplab/droplab.md index 2f8c79abde1..1c6d895b3ab 100644 --- a/doc/development/fe_guide/droplab/droplab.md +++ b/doc/development/fe_guide/droplab/droplab.md @@ -25,6 +25,7 @@ If you do not provide any arguments, it will globally query and instantiate all <!-- ... --> <ul> ``` + ```js const droplab = new DropLab(); droplab.init(); @@ -45,6 +46,7 @@ You can add static list items. <li>Static value 2</li> <ul> ``` + ```js const droplab = new DropLab(); droplab.init(); @@ -62,6 +64,7 @@ a non-global instance of DropLab using the `DropLab.prototype.init` method. <!-- ... --> <ul> ``` + ```js const trigger = document.getElementById('trigger'); const list = document.getElementById('list'); @@ -79,6 +82,7 @@ You can also add hooks to an existing DropLab instance using `DropLab.prototype. <a href="#" id="trigger" data-dropdown-trigger="#list">Toggle</a> <ul id="list" data-dropdown><!-- ... --><ul> ``` + ```js const droplab = new DropLab(); @@ -109,6 +113,7 @@ for all `data-dynamic` dropdown lists tracked by that DropLab instance. <li><a href="#" data-id="{{id}}">{{text}}</a></li> </ul> ``` + ```js const droplab = new DropLab(); @@ -131,6 +136,7 @@ the data as the second argument and the `id` of the trigger element as the first <li><a href="#" data-id="{{id}}">{{text}}</a></li> </ul> ``` + ```js const droplab = new DropLab(); @@ -160,6 +166,7 @@ dropdown lists, one of which is dynamic. </ul> </div> ``` + ```js const droplab = new DropLab(); @@ -216,6 +223,7 @@ Some plugins require configuration values, the config object can be passed as th <a href="#" id="trigger" data-dropdown-trigger="#list">Toggle</a> <ul id="list" data-dropdown><!-- ... --><ul> ``` + ```js const droplab = new DropLab(); diff --git a/doc/development/fe_guide/droplab/plugins/ajax.md b/doc/development/fe_guide/droplab/plugins/ajax.md index b6a883ce6c4..4b76b207d88 100644 --- a/doc/development/fe_guide/droplab/plugins/ajax.md +++ b/doc/development/fe_guide/droplab/plugins/ajax.md @@ -17,18 +17,19 @@ Add the `Ajax` object to the plugins array of a `DropLab.prototype.init` or `Dro <a href="#" id="trigger" data-dropdown-trigger="#list">Toggle</a> <ul id="list" data-dropdown><!-- ... --><ul> ``` + ```js - const droplab = new DropLab(); +const droplab = new DropLab(); - const trigger = document.getElementById('trigger'); - const list = document.getElementById('list'); +const trigger = document.getElementById('trigger'); +const list = document.getElementById('list'); - droplab.addHook(trigger, list, [Ajax], { - Ajax: { - endpoint: '/some-endpoint', - method: 'setData', - }, - }); +droplab.addHook(trigger, list, [Ajax], { + Ajax: { + endpoint: '/some-endpoint', + method: 'setData', + }, +}); ``` Optionally you can set `loadingTemplate` to a HTML string. This HTML string will diff --git a/doc/development/fe_guide/droplab/plugins/filter.md b/doc/development/fe_guide/droplab/plugins/filter.md index 1f188c64fe4..b867394a241 100644 --- a/doc/development/fe_guide/droplab/plugins/filter.md +++ b/doc/development/fe_guide/droplab/plugins/filter.md @@ -17,25 +17,26 @@ Add the `Filter` object to the plugins array of a `DropLab.prototype.init` or `D <li><a href="#" data-id="{{id}}">{{text}}</a></li> <ul> ``` + ```js - const droplab = new DropLab(); - - const trigger = document.getElementById('trigger'); - const list = document.getElementById('list'); - - droplab.init(trigger, list, [Filter], { - Filter: { - template: 'text', - }, - }); - - droplab.addData('trigger', [{ - id: 0, - text: 'Jacob', - }, { - id: 1, - text: 'Jeff', - }]); +const droplab = new DropLab(); + +const trigger = document.getElementById('trigger'); +const list = document.getElementById('list'); + +droplab.init(trigger, list, [Filter], { + Filter: { + template: 'text', + }, +}); + +droplab.addData('trigger', [{ + id: 0, + text: 'Jacob', +}, { + id: 1, + text: 'Jeff', +}]); ``` Above, the input string will be compared against the `test` key of the passed data objects. diff --git a/doc/development/fe_guide/droplab/plugins/input_setter.md b/doc/development/fe_guide/droplab/plugins/input_setter.md index e4050213869..db492da478a 100644 --- a/doc/development/fe_guide/droplab/plugins/input_setter.md +++ b/doc/development/fe_guide/droplab/plugins/input_setter.md @@ -22,33 +22,34 @@ You can also set the `InputSetter` config to an array of objects, which will all <li><a href="#" data-id="{{id}}">{{text}}</a></li> <ul> ``` + ```js - const droplab = new DropLab(); - - const trigger = document.getElementById('trigger'); - const list = document.getElementById('list'); - - const input = document.getElementById('input'); - const div = document.getElementById('div'); - - droplab.init(trigger, list, [InputSetter], { - InputSetter: [{ - input: input, - valueAttribute: 'data-id', - } { - input: div, - valueAttribute: 'data-id', - inputAttribute: 'data-selected-id', - }], - }); - - droplab.addData('trigger', [{ - id: 0, - text: 'Jacob', - }, { - id: 1, - text: 'Jeff', - }]); +const droplab = new DropLab(); + +const trigger = document.getElementById('trigger'); +const list = document.getElementById('list'); + +const input = document.getElementById('input'); +const div = document.getElementById('div'); + +droplab.init(trigger, list, [InputSetter], { + InputSetter: [{ + input: input, + valueAttribute: 'data-id', + } { + input: div, + valueAttribute: 'data-id', + inputAttribute: 'data-selected-id', + }], +}); + +droplab.addData('trigger', [{ + id: 0, + text: 'Jacob', +}, { + id: 1, + text: 'Jeff', +}]); ``` Above, if the second list item was clicked, it would update the `#input` element diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 6c7572352ec..421b7265613 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -34,6 +34,7 @@ new_feature │ └── new_feature_store.js ├── index.js ``` + _For consistency purposes, we recommend you to follow the same structure._ Let's look into each of them: diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 5552d5d37b4..2ade59b76ed 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -237,24 +237,23 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. 1. Create prometheus metrics: ```go - var findAllTagsRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "gitaly_find_all_tags_requests_total", - Help: "Counter of go vs ruby implementation of FindAllTags", - }, - []string{"implementation"}, - ) + var findAllTagsRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitaly_find_all_tags_requests_total", + Help: "Counter of go vs ruby implementation of FindAllTags", + }, + []string{"implementation"}, ) func init() { - prometheus.Register(findAllTagsRequests) + prometheus.Register(findAllTagsRequests) } if featureflag.IsEnabled(ctx, findAllTagsFeatureFlag) { - findAllTagsRequests.WithLabelValues("go").Inc() + findAllTagsRequests.WithLabelValues("go").Inc() // go implementation } else { - findAllTagsRequests.WithLabelValues("ruby").Inc() + findAllTagsRequests.WithLabelValues("ruby").Inc() // ruby implementation } ``` diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 35c5b155594..910d7296057 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -106,32 +106,31 @@ are very appreciative of the work done by translators and proofreaders! 1. Contribute translations to GitLab. See instructions for [translating GitLab](translation.md). - Translating GitLab is a community effort that requires team work and - attention to detail. Proofreaders play an important role helping new - contributors, and ensuring the consistency and quality of translations. - Your conduct and contributions as a translator should reflect this before - requesting to be a proofreader. + Translating GitLab is a community effort that requires team work and + attention to detail. Proofreaders play an important role helping new + contributors, and ensuring the consistency and quality of translations. + Your conduct and contributions as a translator should reflect this before + requesting to be a proofreader. 1. Request proofreader permissions by opening a merge request to add yourself to the list of proofreaders. - Open the [proofreader.md source file][proofreader-src] and click **Edit**. + Open the [proofreader.md source file][proofreader-src] and click **Edit**. - Add your language in alphabetical order, and add yourself to the list - including: - - name - - link to your GitLab profile - - link to your CrowdIn profile + Add your language in alphabetical order, and add yourself to the list + including: + - name + - link to your GitLab profile + - link to your CrowdIn profile - In the merge request description, please include links to any projects you - have previously translated. + In the merge request description, please include links to any projects you + have previously translated. 1. Your request to become a proofreader will be considered on the merits of your previous translations by [GitLab team members](https://about.gitlab.com/team/) or [Core team members](https://about.gitlab.com/core-team/) who are fluent in the language or current proofreaders. - When a request is made for the first proofreader for a language and there are no [GitLab team members](https://about.gitlab.com/team/) - or [Core team members](https://about.gitlab.com/core-team/) who speak the language, we will request links to previous translation work in other communities or projects. - + or [Core team members](https://about.gitlab.com/core-team/) who speak the language, we will request links to previous translation work in other communities or projects. [proofreader-src]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/i18n/proofreader.md diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index 9ba3b922fd8..e1350b02262 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -30,9 +30,11 @@ The following are required to install and test the app: 1. In the **From this URL** field, provide a link to the app descriptor. The host and port must point to your GitLab instance. For example: + ``` https://xxxx.serveo.net/-/jira_connect/app_descriptor.json ``` + 1. Click **Upload**. If the install was successful, you should see the **GitLab for Jira** app under **Manage apps**. diff --git a/doc/development/new_fe_guide/style/prettier.md b/doc/development/new_fe_guide/style/prettier.md index 4495f38f262..5f44c640d76 100644 --- a/doc/development/new_fe_guide/style/prettier.md +++ b/doc/development/new_fe_guide/style/prettier.md @@ -4,7 +4,7 @@ Our code is automatically formatted with [Prettier](https://prettier.io) to foll ## Editor -The easiest way to include prettier in your workflow is by setting up your preferred editor (all major editors are supported) accordingly. We suggest setting up prettier to run automatically when each file is saved. Find [here](https://prettier.io/docs/en/editors.html) the best way to set it up in your preferred editor. +The easiest way to include prettier in your workflow is by setting up your preferred editor (all major editors are supported) accordingly. We suggest setting up prettier to run automatically when each file is saved. Find [here](https://prettier.io/docs/en/editors.html) the best way to set it up in your preferred editor. Please take care that you only let Prettier format the same file types as the global Yarn script does (.js, .vue, and .scss). In VSCode by example you can easily exclude file formats in your settings file: @@ -28,6 +28,7 @@ Updates all currently staged files (based on `git diff`) with Prettier and saves ``` yarn prettier-staged ``` + Checks all currently staged files (based on `git diff`) with Prettier and log which files would need manual updating to the console. ``` diff --git a/doc/development/performance.md b/doc/development/performance.md index c034f4a344b..8b569a677b6 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -246,6 +246,7 @@ irb(main):002:0> results.last.attributes.keys irb(main):003:0> results.where(status: "passed").average(:time).to_s => "0.211340155844156" ``` + These results can also be placed into a PostgreSQL database by setting the `RSPEC_PROFILING_POSTGRES_URL` variable. This is used to profile the test suite when running in the CI environment. diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 59eb3ecfd7e..2dc06ba10a5 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -148,7 +148,7 @@ Once you decided where to put [test environment orchestration scenarios] and the [GitLab QA orchestrator README][gitlab-qa-readme], and [the already existing instance-level scenarios][instance-level scenarios]. -Continued reading: +Continued reading: - [Quick Start Guide](quick_start_guide.md) - [Style Guide](style_guide.md) diff --git a/doc/development/testing_guide/end_to_end/style_guide.md b/doc/development/testing_guide/end_to_end/style_guide.md index 0272e1810f2..52a8116e01c 100644 --- a/doc/development/testing_guide/end_to_end/style_guide.md +++ b/doc/development/testing_guide/end_to_end/style_guide.md @@ -63,17 +63,17 @@ We follow a simple formula roughly based on hungarian notation. - `_checkbox` - `_radio` - `_content` - + *Note: This list is a work in progress. This list will eventually be the end-all enumeration of all available types. I.e., any element that does not end with something in this list is bad form.* - + #### Examples **Good** ```ruby view '...' do - element :edit_button + element :edit_button element :notes_tab element :squash_checkbox element :username_field @@ -84,15 +84,15 @@ end **Bad** ```ruby -view '...' do +view '...' do # `_confirmation` should be `_field`. what sort of confirmation? a checkbox confirmation? no real way to disambiguate. # an appropriate replacement would be `element :password_confirmation_field` element :password_confirmation - # `clone_options` is too vague. If it's a dropdown menu, it should be `clone_dropdown`. + # `clone_options` is too vague. If it's a dropdown menu, it should be `clone_dropdown`. # If it's a checkbox, it should be `clone_checkbox` element :clone_options - + # how is this url being displayed? is it a textbox? a simple span? element :ssh_clone_url end diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 98df0b5ea7c..bb44cc595e9 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -501,17 +501,17 @@ The following are examples of tests that work for both Karma and Jest: it('makes a request', () => { const responseBody = getJSONFixture('some/fixture.json'); // loads spec/javascripts/fixtures/some/fixture.json axiosMock.onGet(endpoint).reply(200, responseBody); - + myButton.click(); - + // ... }); it('uses some HTML element', () => { loadFixtures('some/page.html'); // loads spec/javascripts/fixtures/some/page.html and adds it to the DOM - + const element = document.getElementById('#my-id'); - + // ... }); ``` diff --git a/doc/gitlab-basics/command-line-commands.md b/doc/gitlab-basics/command-line-commands.md index b7e6844f43a..b8ebbbea9d4 100644 --- a/doc/gitlab-basics/command-line-commands.md +++ b/doc/gitlab-basics/command-line-commands.md @@ -139,6 +139,7 @@ pwd ``` clear ``` + ### Sample Git taskflow If you are completely new to Git, looking through some [sample taskflows](https://rogerdudler.github.io/git-guide/) will help you understand best practices for using these commands as you work. diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 73eaf758923..fed3b1ca595 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -59,10 +59,11 @@ Here's a list of the AWS services we will use, with links to pricing information - **ElastiCache**: An in-memory cache environment will be used to provide a High Availability Redis configuration. See the [Amazon ElastiCache pricing](https://aws.amazon.com/elasticache/pricing/). - + NOTE: **Note:** Please note that while we will be using EBS for storage, we do not recommend using EFS as it may negatively impact GitLab's performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. ## Creating an IAM EC2 instance role and profile + To minimize the permissions of the user, we'll create a new [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html) role with limited access: @@ -90,7 +91,7 @@ We'll now create a VPC, a virtual networking environment that you'll control: `10.0.0.0/16`. If you don't require dedicated hardware, you can leave "Tenancy" as default. Click **Yes, Create** when ready. -  +  ### Subnets @@ -107,16 +108,16 @@ RDS instances as well: for example `gitlab-public-10.0.0.0`, select the VPC we created previously, and at the IPv4 CIDR block let's give it a 24 subnet `10.0.0.0/24`: -  +  1. Follow the same steps to create all subnets: - | Name tag | Type |Availability Zone | CIDR block | - | -------- | ---- | ---------------- | ---------- | - | gitlab-public-10.0.0.0 | public | us-west-2a | 10.0.0.0 | - | gitlab-private-10.0.1.0 | private | us-west-2a | 10.0.1.0 | - | gitlab-public-10.0.2.0 | public | us-west-2b | 10.0.2.0 | - | gitlab-private-10.0.3.0 | private | us-west-2b | 10.0.3.0 | + | Name tag | Type |Availability Zone | CIDR block | + | -------- | ---- | ---------------- | ---------- | + | gitlab-public-10.0.0.0 | public | us-west-2a | 10.0.0.0 | + | gitlab-private-10.0.1.0 | private | us-west-2a | 10.0.1.0 | + | gitlab-public-10.0.2.0 | public | us-west-2b | 10.0.2.0 | + | gitlab-private-10.0.3.0 | private | us-west-2b | 10.0.3.0 | ### Route Table @@ -139,7 +140,7 @@ create a new one: 1. Select it from the table, and then under the **Actions** dropdown choose "Attach to VPC". -  +  1. Choose `gitlab-vpc` from the list and hit **Attach**. @@ -154,14 +155,14 @@ it receive traffic from any destination. as destination. In the target, select the `gitlab-gateway` we created previously. Hit **Save** once done. -  +  Next, we must associate the **public** subnets to the route table: 1. Select the **Subnet Associations** tab and hit **Edit**. 1. Check only the public subnet and hit **Save**. -  +  --- @@ -178,12 +179,12 @@ The security group is basically the firewall: Inbound Rules tab. You will need to open the SSH, HTTP, and HTTPS ports. Set the source to `0.0.0.0/0`. -  +  - TIP: **Tip:** - Based on best practices, you should allow SSH traffic from only a known - host or CIDR block. In that case, change the SSH source to be custom and give - it the IP you want to SSH from. + TIP: **Tip:** + Based on best practices, you should allow SSH traffic from only a known + host or CIDR block. In that case, change the SSH source to be custom and give + it the IP you want to SSH from. 1. When done, click **Save**. @@ -204,7 +205,7 @@ create the actual RDS instance. we defined them in the [subnets section](#subnets)). Click **Create** when ready. -  +  ### Creating the database @@ -214,27 +215,27 @@ Now, it's time to create the database: 1. Select PostgreSQL and click **Next**. 1. Since this is a production server, let's choose "Production". Click **Next**. 1. Let's see the instance specifications: - 1. Leave the license model as is (`postgresql-license`). - 1. For the version, select the latest of the 9.6 series (check the - [database requirements](../../install/requirements.md#postgresql-requirements)) - if there are any updates on this). - 1. For the size, let's select a `t2.medium` instance. - 1. Multi-AZ-deployment is recommended as redundancy, so choose "Create - replica in different zone". Read more at - [High Availability (Multi-AZ)](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). - 1. A Provisioned IOPS (SSD) storage type is best suited for HA (though you can - choose a General Purpose (SSD) to reduce the costs). Read more about it at - [Storage for Amazon RDS](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). - -1. The rest of the settings on this page request a DB isntance identifier, username + 1. Leave the license model as is (`postgresql-license`). + 1. For the version, select the latest of the 9.6 series (check the + [database requirements](../../install/requirements.md#postgresql-requirements)) + if there are any updates on this). + 1. For the size, let's select a `t2.medium` instance. + 1. Multi-AZ-deployment is recommended as redundancy, so choose "Create + replica in different zone". Read more at + [High Availability (Multi-AZ)](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). + 1. A Provisioned IOPS (SSD) storage type is best suited for HA (though you can + choose a General Purpose (SSD) to reduce the costs). Read more about it at + [Storage for Amazon RDS](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). + +1. The rest of the settings on this page request a DB instance identifier, username and a master password. We've chosen to use `gitlab-db-ha`, `gitlab` and a very secure password respectively. Keep these in hand for later. 1. Click **Next** to proceed to the advanced settings. 1. Make sure to choose our gitlab VPC, our subnet group, set public accessibility to **No**, and to leave it to create a new security group. The only additional - change which will be helpful is the database name for which we can use - `gitlabhq_production`. At the very bottom, there's an option to enable - auto updates to minor versions. You may want to turn it off. + change which will be helpful is the database name for which we can use + `gitlabhq_production`. At the very bottom, there's an option to enable + auto updates to minor versions. You may want to turn it off. 1. When done, click **Create database**. ### Installing the `pg_trgm` extension for PostgreSQL @@ -276,7 +277,7 @@ To set up Redis: Make sure to select our VPC and its [private subnets](#subnets). Click **Create** when ready. -  +  1. Select **Redis** on the left menu and click **Create** to create a new Redis cluster. Depending on your load, you can choose whether to enable @@ -284,16 +285,16 @@ To set up Redis: chance to deploy Redis in multi availability zones. In this guide, we chose not to enable it. 1. In the settings section: - 1. Give the cluster a name (`gitlab-redis`) and a description. - 1. For the version, select the latest of `3.2` series (e.g., `3.2.10`). - 1. Select the node type and the number of replicas. + 1. Give the cluster a name (`gitlab-redis`) and a description. + 1. For the version, select the latest of `3.2` series (e.g., `3.2.10`). + 1. Select the node type and the number of replicas. 1. In the advanced settings section: 1. Select the multi-AZ auto-failover option. 1. Select the subnet group we created previously. 1. Manually select the preferred availability zones, and under "Replica 2" choose a different zone than the other two. -  +  1. In the security settings, edit the security groups and choose the `gitlab-security-group` we had previously created. @@ -316,11 +317,11 @@ and add a custom TCP rule for port `6379` accessible within itself. On the EC2 dashboard, look for Load Balancer on the left column: 1. Click the **Create Load Balancer** button. - 1. Choose the Application Load Balancer. - 1. Give it a name (`gitlab-loadbalancer`) and set the scheme to "internet-facing". - 1. In the "Listeners" section, make sure it has HTTP and HTTPS. - 1. In the "Availability Zones" section, select the `gitlab-vpc` we have created - and associate the **public subnets**. + 1. Choose the Application Load Balancer. + 1. Give it a name (`gitlab-loadbalancer`) and set the scheme to "internet-facing". + 1. In the "Listeners" section, make sure it has HTTP and HTTPS. + 1. In the "Availability Zones" section, select the `gitlab-vpc` we have created + and associate the **public subnets**. 1. Click **Configure Security Settings** to go to the next section to select the TLS certificate. When done, go to the next step. 1. In the "Security Groups" section, create a new one by giving it a name @@ -355,7 +356,7 @@ Choose the AMI: where `<version>` the latest version as seen on the [releases page](https://about.gitlab.com/releases/). -  +  ### Choose an instance type @@ -504,19 +505,19 @@ The EBS volume will host the Git repositories data: 1. Tell GitLab to store its data in the new directory by editing `/etc/gitlab/gitlab.rb` with your editor: - ```ruby - git_data_dirs({ - "default" => { "path" => "/mnt/gitlab-data" } - }) - ``` + ```ruby + git_data_dirs({ + "default" => { "path" => "/mnt/gitlab-data" } + }) + ``` - where `/mnt/gitlab-data` the location where you will store the Git data. + where `/mnt/gitlab-data` the location where you will store the Git data. 1. Save the file and reconfigure GitLab: - ```sh - sudo gitlab-ctl reconfigure - ``` + ```sh + sudo gitlab-ctl reconfigure + ``` TIP: **Tip:** If you wish to add more than one data volumes to store the Git repositories, @@ -549,15 +550,15 @@ After you SSH into the instance, configure the domain name: 1. Open `/etc/gitlab/gitlab.rb` with your preferred editor. 1. Edit the `external_url` value: - ```ruby - external_url 'http://example.com' - ``` + ```ruby + external_url 'http://example.com' + ``` 1. Reconfigure GitLab: - ```sh - sudo gitlab-ctl reconfigure - ``` + ```sh + sudo gitlab-ctl reconfigure + ``` You should now be able to reach GitLab at the URL you defined. To use HTTPS (recommended), see the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). @@ -608,9 +609,9 @@ To back up GitLab: 1. SSH into your instance. 1. Take a backup: - ```sh - sudo gitlab-rake gitlab:backup:create - ``` + ```sh + sudo gitlab-rake gitlab:backup:create + ``` ### Restoring GitLab from a backup @@ -626,16 +627,16 @@ released, you can update your GitLab instance: 1. SSH into your instance 1. Take a backup: - ```sh - sudo gitlab-rake gitlab:backup:create - ``` + ```sh + sudo gitlab-rake gitlab:backup:create + ``` 1. Update the repositories and install GitLab: - ```sh - sudo apt update - sudo apt install gitlab-ee - ``` + ```sh + sudo apt update + sudo apt install gitlab-ee + ``` After a few minutes, the new version should be up and running. diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index b1f79893baf..c0e1b0ebbc8 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -67,18 +67,19 @@ The first items we need to configure are the basic settings of the underlying vi 1. Enter a `User name` - e.g. **"gitlab-admin"** 1. Select an `Authentication type`, either **SSH public key** or **Password**: - > **Note:** if you're unsure which authentication type to use, select **Password** + > **Note:** if you're unsure which authentication type to use, select **Password** + + 1. If you chose **SSH public key** - enter your `SSH public key` into the field provided + _(read the [SSH documentation][GitLab-Docs-SSH] to learn more about how to set up SSH + public keys)_ + 1. If you chose **Password** - enter the password you wish to use _(this is the password that you + will use later in this tutorial to [SSH] into the VM, so make sure it's a strong password/passphrase)_ - 1. If you chose **SSH public key** - enter your `SSH public key` into the field provided - _(read the [SSH documentation][GitLab-Docs-SSH] to learn more about how to set up SSH - public keys)_ - 1. If you chose **Password** - enter the password you wish to use _(this is the password that you - will use later in this tutorial to [SSH] into the VM, so make sure it's a strong password/passphrase)_ 1. Choose the appropriate `Subscription` tier for your Azure account 1. Choose an existing `Resource Group` or create a new one - e.g. **"GitLab-CE-Azure"** - > **Note:** a "Resource group" is a way to group related resources together for easier administration. - > We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. + > **Note:** a "Resource group" is a way to group related resources together for easier administration. + > We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. 1. Choose a `Location` - if you're unsure, select the default location @@ -248,6 +249,7 @@ rules in the list:  ## Connecting to GitLab + Use the domain name you set up earlier (or the public IP address) to visit your new GitLab instance in your browser. If everything has gone according to plan you should be presented with the following page, asking you to set a _new_ password for the administrator account automatically @@ -348,6 +350,7 @@ your VM, you can use the IP address in its place in the following command: ```bash ssh username@your-azure-domain-name.com ``` + Provide your password at the prompt to authenticate. #### SSH from Windows (PuTTY) @@ -411,12 +414,12 @@ Check out our other [Technical Articles][GitLab-Technical-Articles] or browse th - [GitLab Community Edition][CE] - [GitLab Enterprise Edition][EE] - [Microsoft Azure][Azure] - - [Azure - Free Account FAQ][Azure-Free-Account-FAQ] - - [Azure - Marketplace][Azure-Marketplace] - - [Azure Portal][Azure-Portal] - - [Azure - Pricing Calculator][Azure-Pricing-Calculator] - - [Azure - Troubleshoot SSH Connections to an Azure Linux VM][Azure-Troubleshoot-SSH-Connection] - - [Azure - Properly Shutdown an Azure VM][Azure-Properly-Shutdown-VM] + - [Azure - Free Account FAQ][Azure-Free-Account-FAQ] + - [Azure - Marketplace][Azure-Marketplace] + - [Azure Portal][Azure-Portal] + - [Azure - Pricing Calculator][Azure-Pricing-Calculator] + - [Azure - Troubleshoot SSH Connections to an Azure Linux VM][Azure-Troubleshoot-SSH-Connection] + - [Azure - Properly Shutdown an Azure VM][Azure-Properly-Shutdown-VM] - [SSH], [PuTTY] and [Using SSH in PuTTY][Using-SSH-In-Putty] [Original-Blog-Post]: https://about.gitlab.com/2016/07/13/how-to-setup-a-gitlab-instance-on-microsoft-azure/ "How to Set up a GitLab Instance on Microsoft Azure" diff --git a/doc/install/digitaloceandocker.md b/doc/install/digitaloceandocker.md index 63bb941ad47..b6bf7c95527 100644 --- a/doc/install/digitaloceandocker.md +++ b/doc/install/digitaloceandocker.md @@ -36,30 +36,30 @@ The rest of the steps are identical for macOS and Linux. 1. Login to Digital Ocean. 1. Generate a new API token at <https://cloud.digitalocean.com/settings/api/tokens>. - This command will create a new DO droplet called `gitlab-test-env-do` that will act as a docker host. + This command will create a new DO droplet called `gitlab-test-env-do` that will act as a docker host. - NOTE: **Note:** - 4GB is the minimum requirement for a Docker host that will run more than one GitLab instance. + NOTE: **Note:** + 4GB is the minimum requirement for a Docker host that will run more than one GitLab instance. - - RAM: 4GB - - Name: `gitlab-test-env-do` - - Driver: `digitalocean` + - RAM: 4GB + - Name: `gitlab-test-env-do` + - Driver: `digitalocean` 1. Set the DO token: - ```sh - export DOTOKEN=<your generated token> - ``` + ```sh + export DOTOKEN=<your generated token> + ``` 1. Create the machine: - ```sh - docker-machine create \ - --driver digitalocean \ - --digitalocean-access-token=$DOTOKEN \ - --digitalocean-size "4gb" \ - gitlab-test-env-do - ``` + ```sh + docker-machine create \ + --driver digitalocean \ + --digitalocean-access-token=$DOTOKEN \ + --digitalocean-size "4gb" \ + gitlab-test-env-do + ``` Resource: <https://docs.docker.com/machine/drivers/digital-ocean/>. diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 14bf7012c01..be29bcc7cd7 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -30,16 +30,16 @@ To deploy GitLab on GCP you first need to create a virtual machine: 1. Go to <https://console.cloud.google.com/compute/instances> and log in with your Google credentials. 1. Click on **Create** -  +  -1. On the next page, you can select the type of VM as well as the +1. On the next page, you can select the type of VM as well as the estimated costs. Provide the name of the instance, desired datacenter, and machine type. Note that GitLab recommends at least 2 vCPU's and 4GB of RAM. -  +  1. Click **Change** under Boot disk to select the size, type, and desired operating system. GitLab supports a [variety of linux operating systems][req], including Ubuntu and Debian. Click **Select** when finished. -  +  1. As a last step allow HTTP and HTTPS traffic, then click **Create**. The process will finish in a few seconds. @@ -53,13 +53,13 @@ After a few seconds, the instance will be created and available to log in. The n 1. Click on the SSH button to connect to the instance. 1. A new window will appear, with you logged into the instance. -  +  1. Next, follow the instructions for installing GitLab for the operating system you choose, at <https://about.gitlab.com/install/>. You can use the IP address from the step above, as the hostname. 1. Congratulations! GitLab is now installed and you can access it via your browser. To finish installation, open the URL in your browser and provide the initial administrator password. The username for this account is `root`. -  +  ## Next steps @@ -83,31 +83,31 @@ here's how you configure GitLab to be aware of the change: 1. SSH into the VM. You can easily use the **SSH** button in the Google console and a new window will pop up. -  +  - In the future you might want to set up [connecting with an SSH key][ssh] - instead. + In the future you might want to set up [connecting with an SSH key][ssh] + instead. 1. Edit the config file of Omnibus GitLab using your favorite text editor: - ``` - sudo vim /etc/gitlab/gitlab.rb - ``` + ``` + sudo vim /etc/gitlab/gitlab.rb + ``` 1. Set the `external_url` value to the domain name you wish GitLab to have **without** `https`: - ``` - external_url 'http://gitlab.example.com' - ``` + ``` + external_url 'http://gitlab.example.com' + ``` - We will set up HTTPS in the next step, no need to do this now. + We will set up HTTPS in the next step, no need to do this now. 1. Reconfigure GitLab for the changes to take effect: - ``` - sudo gitlab-ctl reconfigure - ``` + ``` + sudo gitlab-ctl reconfigure + ``` 1. You can now visit GitLab using the domain name. diff --git a/doc/install/installation.md b/doc/install/installation.md index e9206469e5d..06ec00cecc4 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -299,57 +299,57 @@ use of extensions and concurrent index removal, you need at least PostgreSQL 9.2 1. Install the database packages: - ```sh - sudo apt-get install -y postgresql postgresql-client libpq-dev postgresql-contrib - ``` + ```sh + sudo apt-get install -y postgresql postgresql-client libpq-dev postgresql-contrib + ``` 1. Create a database user for GitLab: - ```sh - sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" + ``` 1. Create the `pg_trgm` extension (required for GitLab 8.6+): - ```sh - sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + ``` 1. Create the GitLab production database and grant all privileges on database: - ```sh - sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" + ``` 1. Try connecting to the new database with the new user: - ```sh - sudo -u git -H psql -d gitlabhq_production - ``` + ```sh + sudo -u git -H psql -d gitlabhq_production + ``` 1. Check if the `pg_trgm` extension is enabled: - ```sh - SELECT true AS enabled - FROM pg_available_extensions - WHERE name = 'pg_trgm' - AND installed_version IS NOT NULL; - ``` + ```sh + SELECT true AS enabled + FROM pg_available_extensions + WHERE name = 'pg_trgm' + AND installed_version IS NOT NULL; + ``` - If the extension is enabled this will produce the following output: + If the extension is enabled this will produce the following output: - ``` - enabled - --------- - t - (1 row) - ``` + ``` + enabled + --------- + t + (1 row) + ``` 1. Quit the database session: - ```sh - gitlabhq_production> \q - ``` + ```sh + gitlabhq_production> \q + ``` ## 7. Redis @@ -831,26 +831,27 @@ how to configure GitLab with a relative URL. To use GitLab with HTTPS: 1. In `gitlab.yml`: - 1. Set the `port` option in section 1 to `443`. - 1. Set the `https` option in section 1 to `true`. + 1. Set the `port` option in section 1 to `443`. + 1. Set the `https` option in section 1 to `true`. 1. In the `config.yml` of gitlab-shell: - 1. Set `gitlab_url` option to the HTTPS endpoint of GitLab (e.g. `https://git.example.com`). - 1. Set the certificates using either the `ca_file` or `ca_path` option. + 1. Set `gitlab_url` option to the HTTPS endpoint of GitLab (e.g. `https://git.example.com`). + 1. Set the certificates using either the `ca_file` or `ca_path` option. 1. Use the `gitlab-ssl` Nginx example config instead of the `gitlab` config. - 1. Update `YOUR_SERVER_FQDN`. - 1. Update `ssl_certificate` and `ssl_certificate_key`. - 1. Review the configuration file and consider applying other security and performance enhancing features. + 1. Update `YOUR_SERVER_FQDN`. + 1. Update `ssl_certificate` and `ssl_certificate_key`. + 1. Review the configuration file and consider applying other security and performance enhancing features. Using a self-signed certificate is discouraged but if you must use it, follow the normal directions. Then: 1. Generate a self-signed SSL certificate: - ```sh - mkdir -p /etc/nginx/ssl/ - cd /etc/nginx/ssl/ - sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out gitlab.crt -keyout gitlab.key - sudo chmod o-r gitlab.key - ``` + ```sh + mkdir -p /etc/nginx/ssl/ + cd /etc/nginx/ssl/ + sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out gitlab.crt -keyout gitlab.key + sudo chmod o-r gitlab.key + ``` + 1. In the `config.yml` of gitlab-shell set `self_signed_cert` to `true`. ### Enable Reply by email diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index e4a2d9ecd68..fbbe2a34952 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -70,17 +70,17 @@ In short: 1. Open a terminal and in a new directory run: - ```sh - vagrant init openshift/origin-all-in-one - ``` + ```sh + vagrant init openshift/origin-all-in-one + ``` 1. This will generate a Vagrantfile based on the all-in-one VM image 1. In the same directory where you generated the Vagrantfile enter: - ```sh - vagrant up - ``` + ```sh + vagrant up + ``` This will download the VirtualBox image and fire up the VM with some preconfigured values as you can see in the Vagrantfile. As you may have noticed, you need @@ -195,22 +195,22 @@ In that case, the OpenShift service might not be running, so in order to fix it: 1. SSH into the VM by going to the directory where the Vagrantfile is and then run: - ```sh - vagrant ssh - ``` + ```sh + vagrant ssh + ``` 1. Run `systemctl` and verify by the output that the `openshift` service is not running (it will be in red color). If that's the case start the service with: - ```sh - sudo systemctl start openshift - ``` + ```sh + sudo systemctl start openshift + ``` 1. Verify the service is up with: - ```sh - systemctl status openshift -l - ``` + ```sh + systemctl status openshift -l + ``` Now you will be able to login using `oc` (like we did before) and visit the web console. @@ -393,55 +393,55 @@ Let's see how to do that using the following steps. 1. Make sure you are in the `gitlab` project: - ```sh - oc project gitlab - ``` + ```sh + oc project gitlab + ``` 1. See what services are used for this project: - ```sh - oc get svc - ``` + ```sh + oc get svc + ``` - The output will be similar to: + The output will be similar to: - ``` - NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE - gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d - gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d - gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d - ``` + ``` + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d + gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d + gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d + ``` 1. We need to see the replication controllers of the `gitlab-ce` service. Get a detailed view of the current ones: - ```sh - oc describe rc gitlab-ce - ``` + ```sh + oc describe rc gitlab-ce + ``` - This will return a large detailed list of the current replication controllers. - Search for the name of the GitLab controller, usually `gitlab-ce-1` or if - that failed at some point and you spawned another one, it will be named - `gitlab-ce-2`. + This will return a large detailed list of the current replication controllers. + Search for the name of the GitLab controller, usually `gitlab-ce-1` or if + that failed at some point and you spawned another one, it will be named + `gitlab-ce-2`. 1. Scale GitLab using the previous information: - ```sh - oc scale --replicas=2 replicationcontrollers gitlab-ce-2 - ``` + ```sh + oc scale --replicas=2 replicationcontrollers gitlab-ce-2 + ``` 1. Get the new replicas number to make sure scaling worked: - ```sh - oc get rc gitlab-ce-2 - ``` + ```sh + oc get rc gitlab-ce-2 + ``` - which will return something like: + which will return something like: - ``` - NAME DESIRED CURRENT AGE - gitlab-ce-2 2 2 5d - ``` + ``` + NAME DESIRED CURRENT AGE + gitlab-ce-2 2 2 5d + ``` And that's it! We successfully scaled the replicas to 2 using the CLI. @@ -478,13 +478,13 @@ For OpenShift v3.0, you will need to do this manually: 1. Edit the Security Context: - ```sh - oc edit scc anyuid - ``` + ```sh + oc edit scc anyuid + ``` 1. Add `system:serviceaccount:<project>:gitlab-ce-user` to the `users` section. If you changed the Application Name from the default the user will - will be `<app-name>-user` instead of `gitlab-ce-user` + will be `<app-name>-user` instead of `gitlab-ce-user` 1. Save and exit the editor diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index b53624a33bf..bc6364f57f7 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -58,59 +58,59 @@ assumptions are made: Make sure to follow all steps below: -1. (Optional) If you run short on resources, you can temporarily free up some - memory by shutting down the GitLab service with the following command: +1. (Optional) If you run short on resources, you can temporarily free up some + memory by shutting down the GitLab service with the following command: - ```shell - sudo service gitlab stop - ``` + ```shell + sudo service gitlab stop + ``` -1. Create `/home/git/gitlab/config/initializers/relative_url.rb` +1. Create `/home/git/gitlab/config/initializers/relative_url.rb` - ```shell - cp /home/git/gitlab/config/initializers/relative_url.rb.sample \ - /home/git/gitlab/config/initializers/relative_url.rb - ``` + ```shell + cp /home/git/gitlab/config/initializers/relative_url.rb.sample \ + /home/git/gitlab/config/initializers/relative_url.rb + ``` - and change the following line: + and change the following line: - ```ruby - config.relative_url_root = "/gitlab" - ``` + ```ruby + config.relative_url_root = "/gitlab" + ``` -1. Edit `/home/git/gitlab/config/gitlab.yml` and uncomment/change the - following line: +1. Edit `/home/git/gitlab/config/gitlab.yml` and uncomment/change the + following line: - ```yaml - relative_url_root: /gitlab - ``` + ```yaml + relative_url_root: /gitlab + ``` -1. Edit `/home/git/gitlab/config/unicorn.rb` and uncomment/change the - following line: +1. Edit `/home/git/gitlab/config/unicorn.rb` and uncomment/change the + following line: - ```ruby - ENV['RAILS_RELATIVE_URL_ROOT'] = "/gitlab" - ``` + ```ruby + ENV['RAILS_RELATIVE_URL_ROOT'] = "/gitlab" + ``` -1. Edit `/home/git/gitlab-shell/config.yml` and append the relative path to - the following line: +1. Edit `/home/git/gitlab-shell/config.yml` and append the relative path to + the following line: - ```yaml - gitlab_url: http://127.0.0.1/gitlab - ``` + ```yaml + gitlab_url: http://127.0.0.1/gitlab + ``` -1. Make sure you have copied the supplied init script and the defaults file - as stated in the [installation guide](installation.md#install-init-script). - Then, edit `/etc/default/gitlab` and set in `gitlab_workhorse_options` the - `-authBackend` setting to read like: +1. Make sure you have copied the supplied init script and the defaults file + as stated in the [installation guide](installation.md#install-init-script). + Then, edit `/etc/default/gitlab` and set in `gitlab_workhorse_options` the + `-authBackend` setting to read like: - ```shell - -authBackend http://127.0.0.1:8080/gitlab - ``` + ```shell + -authBackend http://127.0.0.1:8080/gitlab + ``` - **Note:** - If you are using a custom init script, make sure to edit the above - gitlab-workhorse setting as needed. + **Note:** + If you are using a custom init script, make sure to edit the above + gitlab-workhorse setting as needed. 1. [Restart GitLab][] for the changes to take effect. @@ -118,9 +118,9 @@ Make sure to follow all steps below: To disable the relative URL: -1. Remove `/home/git/gitlab/config/initializers/relative_url.rb` +1. Remove `/home/git/gitlab/config/initializers/relative_url.rb` -1. Follow the same as above starting from 2. and set up the +1. Follow the same as above starting from 2. and set up the GitLab URL to one that doesn't contain a relative path. [omnibus-rel]: https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab "How to set up relative URL in Omnibus GitLab" diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index c67375ede50..5061b863e79 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -16,64 +16,64 @@ application. 1. At the top of the Settings screen, you should see your Domain, Client ID and Client Secret. Take note of these as you'll need to put them in the configuration file. For example: - - Domain: `test1234.auth0.com` - - Client ID: `t6X8L2465bNePWLOvt9yi41i` - - Client Secret: `KbveM3nqfjwCbrhaUy_gDu2dss8TIlHIdzlyf33pB7dEK5u_NyQdp65O_o02hXs2` + - Domain: `test1234.auth0.com` + - Client ID: `t6X8L2465bNePWLOvt9yi41i` + - Client Secret: `KbveM3nqfjwCbrhaUy_gDu2dss8TIlHIdzlyf33pB7dEK5u_NyQdp65O_o02hXs2` 1. Fill in the Allowed Callback URLs: - - `http://YOUR_GITLAB_URL/users/auth/auth0/callback` (or) - - `https://YOUR_GITLAB_URL/users/auth/auth0/callback` + - `http://YOUR_GITLAB_URL/users/auth/auth0/callback` (or) + - `https://YOUR_GITLAB_URL/users/auth/auth0/callback` 1. Fill in the Allowed Origins (CORS): - - `http://YOUR_GITLAB_URL` (or) - - `https://YOUR_GITLAB_URL` + - `http://YOUR_GITLAB_URL` (or) + - `https://YOUR_GITLAB_URL` 1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - For omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "auth0", - "args" => { client_id: 'YOUR_AUTH0_CLIENT_ID', - client_secret: 'YOUR_AUTH0_CLIENT_SECRET', - domain: 'YOUR_AUTH0_DOMAIN', - scope: 'openid profile email' - } - } - ] - ``` - - For installations from source: - - ```yaml - - { name: 'auth0', - args: { - client_id: 'YOUR_AUTH0_CLIENT_ID', - client_secret: 'YOUR_AUTH0_CLIENT_SECRET', - domain: 'YOUR_AUTH0_DOMAIN', - scope: 'openid profile email' } - } - ``` + For omnibus package: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "auth0", + "args" => { client_id: 'YOUR_AUTH0_CLIENT_ID', + client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + domain: 'YOUR_AUTH0_DOMAIN', + scope: 'openid profile email' + } + } + ] + ``` + + For installations from source: + + ```yaml + - { name: 'auth0', + args: { + client_id: 'YOUR_AUTH0_CLIENT_ID', + client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + domain: 'YOUR_AUTH0_DOMAIN', + scope: 'openid profile email' } + } + ``` 1. Change `YOUR_AUTH0_CLIENT_ID` to the client ID from the Auth0 Console page from step 5. @@ -81,8 +81,8 @@ application. 1. Change `YOUR_AUTH0_CLIENT_SECRET` to the client secret from the Auth0 Console page from step 5. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be an Auth0 icon below the regular sign in form. Click the icon to begin the authentication process. Auth0 will ask the diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 68ec8c4b5c2..5d8f2ebcb8b 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -30,97 +30,97 @@ To enable the Bitbucket OmniAuth provider you must register your application with Bitbucket.org. Bitbucket will generate an application ID and secret key for you to use. -1. Sign in to [Bitbucket.org](https://bitbucket.org). -1. Navigate to your individual user settings (**Bitbucket settings**) or a team's - settings (**Manage team**), depending on how you want the application registered. - It does not matter if the application is registered as an individual or a - team, that is entirely up to you. -1. Select **OAuth** in the left menu under "Access Management". -1. Select **Add consumer**. -1. Provide the required details: - - | Item | Description | - | :--- | :---------- | - | **Name** | This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. | - | **Application description** | Fill this in if you wish. | - | **Callback URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com/users/auth`. | - | **URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com`. | - - NOTE: Be sure to append `/users/auth` to the end of the callback URL - to prevent a [OAuth2 convert - redirect](http://tetraph.com/covert_redirect/) vulnerability. - - NOTE: Starting in GitLab 8.15, you MUST specify a callback URL, or you will - see an "Invalid redirect_uri" message. For more details, see [the - Bitbucket documentation](https://confluence.atlassian.com/bitbucket/oauth-faq-338365710.html). - - And grant at least the following permissions: - - ``` - Account: Email, Read - Projects: Read - Repositories: Read - Pull Requests: Read - Issues: Read - Wiki: Read and Write - ``` - -  - -1. Select **Save**. -1. Select your newly created OAuth consumer and you should now see a Key and - Secret in the list of OAuth consumers. Keep this page open as you continue - the configuration. - -  - -1. On your GitLab server, open the configuration file: - - ``` - # For Omnibus packages - sudo editor /etc/gitlab/gitlab.rb - - # For installations from source - sudo -u git -H editor /home/git/gitlab/config/gitlab.yml - ``` - -1. Add the Bitbucket provider configuration: - - For Omnibus packages: - - ```ruby - gitlab_rails['omniauth_enabled'] = true - - gitlab_rails['omniauth_providers'] = [ - { - "name" => "bitbucket", - "app_id" => "BITBUCKET_APP_KEY", - "app_secret" => "BITBUCKET_APP_SECRET", - "url" => "https://bitbucket.org/" - } - ] - ``` - - For installations from source: - - ```yaml - omniauth: - enabled: true - providers: - - { name: 'bitbucket', - app_id: 'BITBUCKET_APP_KEY', - app_secret: 'BITBUCKET_APP_SECRET', - url: 'https://bitbucket.org/' } - ``` - - --- - - Where `BITBUCKET_APP_KEY` is the Key and `BITBUCKET_APP_SECRET` the Secret - from the Bitbucket application page. - -1. Save the configuration file. -1. For the changes to take effect, [reconfigure GitLab][] if you installed via - Omnibus, or [restart][] if installed from source. +1. Sign in to [Bitbucket.org](https://bitbucket.org). +1. Navigate to your individual user settings (**Bitbucket settings**) or a team's + settings (**Manage team**), depending on how you want the application registered. + It does not matter if the application is registered as an individual or a + team, that is entirely up to you. +1. Select **OAuth** in the left menu under "Access Management". +1. Select **Add consumer**. +1. Provide the required details: + + | Item | Description | + | :--- | :---------- | + | **Name** | This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. | + | **Application description** | Fill this in if you wish. | + | **Callback URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com/users/auth`. | + | **URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com`. | + + NOTE: Be sure to append `/users/auth` to the end of the callback URL + to prevent a [OAuth2 convert + redirect](http://tetraph.com/covert_redirect/) vulnerability. + + NOTE: Starting in GitLab 8.15, you MUST specify a callback URL, or you will + see an "Invalid redirect_uri" message. For more details, see [the + Bitbucket documentation](https://confluence.atlassian.com/bitbucket/oauth-faq-338365710.html). + + And grant at least the following permissions: + + ``` + Account: Email, Read + Projects: Read + Repositories: Read + Pull Requests: Read + Issues: Read + Wiki: Read and Write + ``` + +  + +1. Select **Save**. +1. Select your newly created OAuth consumer and you should now see a Key and + Secret in the list of OAuth consumers. Keep this page open as you continue + the configuration. + +  + +1. On your GitLab server, open the configuration file: + + ``` + # For Omnibus packages + sudo editor /etc/gitlab/gitlab.rb + + # For installations from source + sudo -u git -H editor /home/git/gitlab/config/gitlab.yml + ``` + +1. Add the Bitbucket provider configuration: + + For Omnibus packages: + + ```ruby + gitlab_rails['omniauth_enabled'] = true + + gitlab_rails['omniauth_providers'] = [ + { + "name" => "bitbucket", + "app_id" => "BITBUCKET_APP_KEY", + "app_secret" => "BITBUCKET_APP_SECRET", + "url" => "https://bitbucket.org/" + } + ] + ``` + + For installations from source: + + ```yaml + omniauth: + enabled: true + providers: + - { name: 'bitbucket', + app_id: 'BITBUCKET_APP_KEY', + app_secret: 'BITBUCKET_APP_SECRET', + url: 'https://bitbucket.org/' } + ``` + + --- + + Where `BITBUCKET_APP_KEY` is the Key and `BITBUCKET_APP_SECRET` the Secret + from the Bitbucket application page. + +1. Save the configuration file. +1. For the changes to take effect, [reconfigure GitLab][] if you installed via + Omnibus, or [restart][] if installed from source. On the sign in page there should now be a Bitbucket icon below the regular sign in form. Click the icon to begin the authentication process. Bitbucket will ask diff --git a/doc/integration/cas.md b/doc/integration/cas.md index c6178fa44f0..f99337376a8 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -2,63 +2,63 @@ To enable the CAS OmniAuth provider you must register your application with your CAS instance. This requires the service URL GitLab will supply to CAS. It should be something like: `https://gitlab.example.com:443/users/auth/cas3/callback?url`. By default handling for SLO is enabled, you only need to configure CAS for backchannel logout. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name"=> "cas3", - "label"=> "cas", - "args"=> { - "url"=> 'CAS_SERVER', - "login_url"=> '/CAS_PATH/login', - "service_validate_url"=> '/CAS_PATH/p3/serviceValidate', - "logout_url"=> '/CAS_PATH/logout' - } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name"=> "cas3", + "label"=> "cas", + "args"=> { + "url"=> 'CAS_SERVER', + "login_url"=> '/CAS_PATH/login', + "service_validate_url"=> '/CAS_PATH/p3/serviceValidate', + "logout_url"=> '/CAS_PATH/logout' + } + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'cas3', - label: 'cas', - args: { - url: 'CAS_SERVER', - login_url: '/CAS_PATH/login', - service_validate_url: '/CAS_PATH/p3/serviceValidate', - logout_url: '/CAS_PATH/logout'} } - ``` + ``` + - { name: 'cas3', + label: 'cas', + args: { + url: 'CAS_SERVER', + login_url: '/CAS_PATH/login', + service_validate_url: '/CAS_PATH/p3/serviceValidate', + logout_url: '/CAS_PATH/logout'} } + ``` -1. Change 'CAS_PATH' to the root of your CAS instance (ie. `cas`). +1. Change 'CAS_PATH' to the root of your CAS instance (ie. `cas`). -1. If your CAS instance does not use default TGC lifetimes, update the `cas3.session_duration` to at least the current TGC maximum lifetime. To explicitly disable SLO, regardless of CAS settings, set this to 0. +1. If your CAS instance does not use default TGC lifetimes, update the `cas3.session_duration` to at least the current TGC maximum lifetime. To explicitly disable SLO, regardless of CAS settings, set this to 0. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a CAS tab in the sign in form. diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index da1df07a75d..fff06254da7 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -12,6 +12,7 @@ special searches: - [Advanced Syntax Search](../user/search/advanced_search_syntax.md) ## Version Requirements + <!-- Please remember to update ee/lib/system_check/app/elasticsearch_check.rb if this changes --> | GitLab version | Elasticsearch version | @@ -424,91 +425,90 @@ Here are some common pitfalls and how to overcome them: - **How can I verify my GitLab instance is using Elasticsearch?** - The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: + The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: - ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term'}) - pp s.search_objects.class.name - ``` + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term'}) + pp s.search_objects.class.name + ``` - If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. + If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. - **I updated GitLab and now I can't find anything** - We continuously make updates to our indexing strategies and aim to support - newer versions of Elasticsearch. When indexing changes are made, it may - be necessary for you to [reindex](#adding-gitlabs-data-to-the-elasticsearch-index) after updating GitLab. + We continuously make updates to our indexing strategies and aim to support + newer versions of Elasticsearch. When indexing changes are made, it may + be necessary for you to [reindex](#adding-gitlabs-data-to-the-elasticsearch-index) after updating GitLab. - **I indexed all the repositories but I can't find anything** - Make sure you indexed all the database data [as stated above](#adding-gitlabs-data-to-the-elasticsearch-index). + Make sure you indexed all the database data [as stated above](#adding-gitlabs-data-to-the-elasticsearch-index). - Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. + Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. - If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): + If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): - ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term', :scope => ‘blobs’}) - pp s.search_objects.to_a - ``` + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term', :scope => ‘blobs’}) + pp s.search_objects.to_a + ``` - See [Elasticsearch Index Scopes](elasticsearch.md#elasticsearch-index-scopes) for more information on searching for specific types of data. + See [Elasticsearch Index Scopes](elasticsearch.md#elasticsearch-index-scopes) for more information on searching for specific types of data. - **I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything** - You will need to re-run all the rake tasks to re-index the database, repositories, and wikis. + You will need to re-run all the rake tasks to re-index the database, repositories, and wikis. - **The indexing process is taking a very long time** - The more data present in your GitLab instance, the longer the indexing process takes. + The more data present in your GitLab instance, the longer the indexing process takes. - **No new data is added to the Elasticsearch index when I push code** - When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could - happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, - run the `gitlab:elastic:clear_locked_projects` rake task. + When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could + happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, + run the `gitlab:elastic:clear_locked_projects` rake task. - **"Can't specify parent if no parent field has been configured"** - If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get - exception in lots of different cases: - - ```text - Elasticsearch::Transport::Transport::Errors::BadRequest([400] { - "error": { - "root_cause": [{ - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }], - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }, - "status": 400 - }): - ``` - - This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, - see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/update/8.11-to-8.12.md#11-elasticsearch-index-update-if-you-currently-use-elasticsearch). + If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get + exception in lots of different cases: + + ```text + Elasticsearch::Transport::Transport::Errors::BadRequest([400] { + "error": { + "root_cause": [{ + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }], + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }, + "status": 400 + }): + ``` + + This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, + see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/update/8.11-to-8.12.md#11-elasticsearch-index-update-if-you-currently-use-elasticsearch). - Exception `Elasticsearch::Transport::Transport::Errors::BadRequest` - If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). - There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. + If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). + There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. - Exception `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` - ```text - [413] {"Message":"Request size exceeded 10485760 bytes"} - ``` - - This exception is seen when your Elasticsearch cluster is configured to reject - requests above a certain size (10MiB in this case). This corresponds to the - `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a - larger size and restart your Elasticsearch cluster. + ```text + [413] {"Message":"Request size exceeded 10485760 bytes"} + ``` - AWS has [fixed limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html) - for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of - the underlying instance. + This exception is seen when your Elasticsearch cluster is configured to reject + requests above a certain size (10MiB in this case). This corresponds to the + `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a + larger size and restart your Elasticsearch cluster. + AWS has [fixed limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html) + for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of + the underlying instance. diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index fe789a80eed..837434da737 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -2,7 +2,7 @@ To enable the Facebook OmniAuth provider you must register your application with Facebook. Facebook will generate an app ID and secret key for you to use. -1. Sign in to the [Facebook Developer Platform](https://developers.facebook.com/). +1. Sign in to the [Facebook Developer Platform](https://developers.facebook.com/). 1. Choose "My Apps" > "Add a New App" @@ -47,53 +47,53 @@ To enable the Facebook OmniAuth provider you must register your application with  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "facebook", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET" - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "facebook", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET" + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'facebook', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET' } - ``` + ``` + - { name: 'facebook', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET' } + ``` -1. Change 'YOUR_APP_ID' to the API key from Facebook page in step 10. +1. Change 'YOUR_APP_ID' to the API key from Facebook page in step 10. -1. Change 'YOUR_APP_SECRET' to the API secret from the Facebook page in step 10. +1. Change 'YOUR_APP_SECRET' to the API secret from the Facebook page in step 10. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Facebook icon below the regular sign in form. Click the icon to begin the authentication process. Facebook will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. diff --git a/doc/integration/github.md b/doc/integration/github.md index 5b01dd9feb7..c8dbae65465 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -7,111 +7,111 @@ You can integrate your GitLab instance with GitHub.com as well as GitHub Enterpr To enable GitHub OmniAuth provider, you must use GitHub's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must register an application as an OAuth App on GitHub. -1. Sign in to GitHub. +1. Sign in to GitHub. -1. Navigate to your individual user or organization settings, depending on how you want the application registered. It does not matter if the application is registered as an individual or an organization - that is entirely up to you. +1. Navigate to your individual user or organization settings, depending on how you want the application registered. It does not matter if the application is registered as an individual or an organization - that is entirely up to you. - - For individual accounts, select **Developer settings** from the left menu, then select **OAuth Apps**. - - For organization accounts, directly select **OAuth Apps** from the left menu. + - For individual accounts, select **Developer settings** from the left menu, then select **OAuth Apps**. + - For organization accounts, directly select **OAuth Apps** from the left menu. -1. Select **Register an application** (if you don't have any OAuth App) or **New OAuth App** (if you already have OAuth Apps). -  +1. Select **Register an application** (if you don't have any OAuth App) or **New OAuth App** (if you already have OAuth Apps). +  -1. Provide the required details. - - Application name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. - - Homepage URL: The URL of your GitLab installation. For example, `https://gitlab.example.com`. - - Application description: Fill this in if you wish. - - Authorization callback URL: `http(s)://${YOUR_DOMAIN}/users/auth`. Please make sure the port is included if your GitLab instance is not configured on default port. -  +1. Provide the required details. + - Application name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. + - Homepage URL: The URL of your GitLab installation. For example, `https://gitlab.example.com`. + - Application description: Fill this in if you wish. + - Authorization callback URL: `http(s)://${YOUR_DOMAIN}/users/auth`. Please make sure the port is included if your GitLab instance is not configured on default port. +  - NOTE: Be sure to append `/users/auth` to the end of the callback URL - to prevent a [OAuth2 convert - redirect](http://tetraph.com/covert_redirect/) vulnerability. + NOTE: Be sure to append `/users/auth` to the end of the callback URL + to prevent a [OAuth2 convert + redirect](http://tetraph.com/covert_redirect/) vulnerability. -1. Select **Register application**. +1. Select **Register application**. -1. You should now see a pair of **Client ID** and **Client Secret** near the top right of the page (see screenshot). - Keep this page open as you continue configuration. -  +1. You should now see a pair of **Client ID** and **Client Secret** near the top right of the page (see screenshot). + Keep this page open as you continue configuration. +  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - For GitHub.com: + For GitHub.com: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "scope" => "user:email" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "scope" => "user:email" } + } + ] + ``` - For GitHub Enterprise: + For GitHub Enterprise: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "url" => "https://github.example.com/", - "args" => { "scope" => "user:email" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "url" => "https://github.example.com/", + "args" => { "scope" => "user:email" } + } + ] + ``` - For installation from source: + For installation from source: - For GitHub.com: + For GitHub.com: - ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'user:email' } } - ``` + ``` + - { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'user:email' } } + ``` - For GitHub Enterprise: + For GitHub Enterprise: - ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - args: { scope: 'user:email' } } - ``` + ``` + - { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + args: { scope: 'user:email' } } + ``` - __Replace `https://github.example.com/` with your GitHub URL.__ + __Replace `https://github.example.com/` with your GitHub URL.__ -1. Change `YOUR_APP_ID` to the Client ID from the GitHub application page from step 6. +1. Change `YOUR_APP_ID` to the Client ID from the GitHub application page from step 6. -1. Change `YOUR_APP_SECRET` to the Client Secret from the GitHub application page from step 6. +1. Change `YOUR_APP_SECRET` to the Client Secret from the GitHub application page from step 6. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure GitLab][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure GitLab][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a GitHub icon below the regular sign in form. Click the icon to begin the authentication process. GitHub will ask the user to sign in and authorize the GitLab application. @@ -127,16 +127,16 @@ and changing the global Git `sslVerify` option to `false` in the GitLab server. For omnibus package: ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "url" => "https://github.example.com/", - "verify_ssl" => false, - "args" => { "scope" => "user:email" } - } - ] +gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "url" => "https://github.example.com/", + "verify_ssl" => false, + "args" => { "scope" => "user:email" } + } +] ``` You will also need to disable Git SSL verification on the server hosting GitLab. @@ -148,11 +148,11 @@ omnibus_gitconfig['system'] = { "http" => ["sslVerify = false"] } For installation from source: ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - verify_ssl: false, - args: { scope: 'user:email' } } +- { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + verify_ssl: false, + args: { scope: 'user:email' } } ``` You will also need to disable Git SSL verification on the server hosting GitLab. diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 70087576678..46da3d88d90 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -5,78 +5,78 @@ Import projects from GitLab.com and login to your GitLab instance with your GitL To enable the GitLab.com OmniAuth provider you must register your application with GitLab.com. GitLab.com will generate an application ID and secret key for you to use. -1. Sign in to GitLab.com +1. Sign in to GitLab.com 1. On the upper right corner, click on your avatar and go to your **Settings**. -1. Select **Applications** in the left menu. +1. Select **Applications** in the left menu. -1. Provide the required details for **Add new application**. - - Name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. - - Redirect URI: +1. Provide the required details for **Add new application**. + - Name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. + - Redirect URI: - ``` - http://your-gitlab.example.com/import/gitlab/callback - http://your-gitlab.example.com/users/auth/gitlab/callback - ``` + ``` + http://your-gitlab.example.com/import/gitlab/callback + http://your-gitlab.example.com/users/auth/gitlab/callback + ``` - The first link is required for the importer and second for the authorization. + The first link is required for the importer and second for the authorization. -1. Select **Save application**. +1. Select **Save application**. -1. You should now see a **Application Id** and **Secret** near the top right of the page (see screenshot). - Keep this page open as you continue configuration. -  +1. You should now see a **Application Id** and **Secret** near the top right of the page (see screenshot). + Keep this page open as you continue configuration. +  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "gitlab", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "scope" => "api" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "gitlab", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "scope" => "api" } + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'gitlab', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'api' } } - ``` + ``` + - { name: 'gitlab', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'api' } } + ``` -1. Change 'YOUR_APP_ID' to the Application ID from the GitLab.com application page. +1. Change 'YOUR_APP_ID' to the Application ID from the GitLab.com application page. -1. Change 'YOUR_APP_SECRET' to the secret from the GitLab.com application page. +1. Change 'YOUR_APP_SECRET' to the secret from the GitLab.com application page. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a GitLab.com icon below the regular sign in form. Click the icon to begin the authentication process. GitLab.com will ask the user to sign in and authorize the GitLab application. diff --git a/doc/integration/google.md b/doc/integration/google.md index d2b4e119978..4f6999571b6 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -10,10 +10,10 @@ In Google's side: 1. Navigate to the [cloud resource manager](https://console.cloud.google.com/cloud-resource-manager) page 1. Select **Create Project** 1. Provide the project information: - - **Project name** - "GitLab" works just fine here. - - **Project ID** - Must be unique to all Google Developer registered applications. - Google provides a randomly generated Project ID by default. You can use - the randomly generated ID or choose a new one. + - **Project name** - "GitLab" works just fine here. + - **Project ID** - Must be unique to all Google Developer registered applications. + Google provides a randomly generated Project ID by default. You can use + the randomly generated ID or choose a new one. 1. Refresh the page and you should see your new project in the list 1. Go to the [Google API Console](https://console.developers.google.com/apis/dashboard) 1. Select the previously created project form the upper left corner @@ -21,17 +21,17 @@ In Google's side: 1. Select **OAuth consent screen** and fill the form with the required information 1. In the **Credentials** tab, select **Create credentials > OAuth client ID** 1. Fill in the required information - - **Application type** - Choose "Web Application" - - **Name** - Use the default one or provide your own - - **Authorized JavaScript origins** -This isn't really used by GitLab but go - ahead and put `https://gitlab.example.com` - - **Authorized redirect URIs** - Enter your domain name followed by the - callback URIs one at a time: - - ``` - https://gitlab.example.com/users/auth/google_oauth2/callback - https://gitlab.example.com/-/google_api/auth/callback - ``` + - **Application type** - Choose "Web Application" + - **Name** - Use the default one or provide your own + - **Authorized JavaScript origins** -This isn't really used by GitLab but go + ahead and put `https://gitlab.example.com` + - **Authorized redirect URIs** - Enter your domain name followed by the + callback URIs one at a time: + + ``` + https://gitlab.example.com/users/auth/google_oauth2/callback + https://gitlab.example.com/-/google_api/auth/callback + ``` 1. You should now be able to see a Client ID and Client secret. Note them down or keep this page open as you will need them later. @@ -45,64 +45,64 @@ On your GitLab server: 1. Open the configuration file. - For Omnibus GitLab: + For Omnibus GitLab: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - For Omnibus GitLab: + For Omnibus GitLab: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "google_oauth2", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "access_type" => "offline", "approval_prompt" => '' } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "google_oauth2", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "access_type" => "offline", "approval_prompt" => '' } + } + ] + ``` - For installations from source: + For installations from source: - ```yaml - - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { access_type: 'offline', approval_prompt: '' } } - ``` + ```yaml + - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { access_type: 'offline', approval_prompt: '' } } + ``` 1. Change `YOUR_APP_ID` to the client ID from the Google Developer page 1. Similarly, change `YOUR_APP_SECRET` to the client secret 1. Make sure that you configure GitLab to use an FQDN as Google will not accept raw IP addresses. - For Omnibus packages: + For Omnibus packages: - ```ruby - external_url 'https://gitlab.example.com' - ``` + ```ruby + external_url 'https://gitlab.example.com' + ``` - For installations from source: + For installations from source: - ```yaml - gitlab: - host: https://gitlab.example.com - ``` + ```yaml + gitlab: + host: https://gitlab.example.com + ``` -1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Save the configuration file. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Google icon below the regular sign in form. Click the icon to begin the authentication process. Google will ask the diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index cf2ef511264..b4f2025265e 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -50,20 +50,20 @@ For source installations, make sure the `kerberos` gem group authentication. In most cases, you only need to enable Kerberos and specify the location of the keytab: - ```yaml - omniauth: - enabled: true - allow_single_sign_on: ['kerberos'] + ```yaml + omniauth: + enabled: true + allow_single_sign_on: ['kerberos'] - kerberos: - # Allow the HTTP Negotiate authentication method for Git clients - enabled: true + kerberos: + # Allow the HTTP Negotiate authentication method for Git clients + enabled: true - # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user, - # and should be different from other keytabs in the system. - # (default: use default keytab from Krb5 config) - keytab: /etc/http.keytab - ``` + # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user, + # and should be different from other keytabs in the system. + # (default: use default keytab from Krb5 config) + keytab: /etc/http.keytab + ``` 1. [Restart GitLab] for the changes to take effect. @@ -73,13 +73,13 @@ For source installations, make sure the `kerberos` gem group 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['omniauth_enabled'] = true - gitlab_rails['omniauth_allow_single_sign_on'] = ['kerberos'] + ```ruby + gitlab_rails['omniauth_enabled'] = true + gitlab_rails['omniauth_allow_single_sign_on'] = ['kerberos'] - gitlab_rails['kerberos_enabled'] = true - gitlab_rails['kerberos_keytab'] = "/etc/http.keytab" - ``` + gitlab_rails['kerberos_enabled'] = true + gitlab_rails['kerberos_keytab'] = "/etc/http.keytab" + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -149,26 +149,26 @@ keep offering only `basic` authentication. (e.g., `/etc/nginx/sites-available/gitlab-ssl`) and configure NGINX to listen to port `8443` in addition to the standard HTTPS port: - ```conf - server { - listen 0.0.0.0:443 ssl; - listen [::]:443 ipv6only=on ssl default_server; - listen 0.0.0.0:8443 ssl; - listen [::]:8443 ipv6only=on ssl; - ``` + ```conf + server { + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl default_server; + listen 0.0.0.0:8443 ssl; + listen [::]:8443 ipv6only=on ssl; + ``` 1. Update the Kerberos section of [gitlab.yml]: - ```yaml - kerberos: - # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails. - # To support both Basic and Negotiate methods with older versions of Git, configure - # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines - # to dedicate this port to Kerberos authentication. (default: false) - use_dedicated_port: true - port: 8443 - https: true - ``` + ```yaml + kerberos: + # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails. + # To support both Basic and Negotiate methods with older versions of Git, configure + # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines + # to dedicate this port to Kerberos authentication. (default: false) + use_dedicated_port: true + port: 8443 + https: true + ``` 1. [Restart GitLab] and NGINX for the changes to take effect. @@ -178,11 +178,11 @@ keep offering only `basic` authentication. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['kerberos_use_dedicated_port'] = true - gitlab_rails['kerberos_port'] = 8443 - gitlab_rails['kerberos_https'] = true - ``` + ```ruby + gitlab_rails['kerberos_use_dedicated_port'] = true + gitlab_rails['kerberos_port'] = 8443 + gitlab_rails['kerberos_https'] = true + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -214,12 +214,12 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / 1. Edit [gitlab.yml] and remove the `- { name: 'kerberos' }` line under omniauth providers: - ```yaml - omniauth: - # ... - providers: - - { name: 'kerberos' } # <-- remove this line - ``` + ```yaml + omniauth: + # ... + providers: + - { name: 'kerberos' } # <-- remove this line + ``` 1. [Restart GitLab] for the changes to take effect. @@ -230,11 +230,11 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / 1. Edit `/etc/gitlab/gitlab.rb` and remove the `{ "name" => "kerberos" }` line under `gitlab_rails['omniauth_providers']`: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { "name" => "kerberos" } # <-- remove this entry - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { "name" => "kerberos" } # <-- remove this entry + ] + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -290,7 +290,7 @@ remote: HTTP Basic: Access denied fatal: Authentication failed for '<KRB5 path>' ``` -If you are using Git v2.11 or newer and see the above error when cloning, you can +If you are using Git v2.11 or newer and see the above error when cloning, you can set the `http.emptyAuth` Git option to `true` to fix this: ``` diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index 3c1a0f2a117..f4119b1d1ce 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -24,11 +24,11 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc 1. Register your application in the OAuth2 provider you wish to authenticate with. - The redirect URI you provide when registering the application should be: + The redirect URI you provide when registering the application should be: - ``` - http://your-gitlab.host.com/users/auth/oauth2_generic/callback - ``` + ``` + http://your-gitlab.host.com/users/auth/oauth2_generic/callback + ``` 1. You should now be able to get a Client ID and Client Secret. Where this shows up will differ for each provider. @@ -36,18 +36,18 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc 1. On your GitLab server, open the configuration file. - For Omnibus package: + For Omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index bf5debc7694..7a92ed994c7 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -71,57 +71,57 @@ To change these settings: - **For omnibus package** - Open the configuration file: + Open the configuration file: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - and change: + and change: - ```ruby - # Versions prior to 11.4 require this to be set to true - # gitlab_rails['omniauth_enabled'] = nil + ```ruby + # Versions prior to 11.4 require this to be set to true + # gitlab_rails['omniauth_enabled'] = nil - # CAUTION! - # This allows users to login without having a user account first. Define the allowed providers - # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. - # User accounts will be created automatically when authentication was successful. - gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'twitter'] - gitlab_rails['omniauth_auto_link_ldap_user'] = true - gitlab_rails['omniauth_block_auto_created_users'] = true - ``` + # CAUTION! + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. + # User accounts will be created automatically when authentication was successful. + gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'twitter'] + gitlab_rails['omniauth_auto_link_ldap_user'] = true + gitlab_rails['omniauth_block_auto_created_users'] = true + ``` - **For installations from source** - Open the configuration file: + Open the configuration file: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` - and change the following section: + and change the following section: - ```yaml - ## OmniAuth settings - omniauth: - # Allow login via Twitter, Google, etc. using OmniAuth providers - # Versions prior to 11.4 require this to be set to true - # enabled: true + ```yaml + ## OmniAuth settings + omniauth: + # Allow login via Twitter, Google, etc. using OmniAuth providers + # Versions prior to 11.4 require this to be set to true + # enabled: true - # CAUTION! - # This allows users to login without having a user account first. Define the allowed providers - # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. - # User accounts will be created automatically when authentication was successful. - allow_single_sign_on: ["saml", "twitter"] + # CAUTION! + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. + # User accounts will be created automatically when authentication was successful. + allow_single_sign_on: ["saml", "twitter"] - auto_link_ldap_user: true + auto_link_ldap_user: true - # Locks down those users until they have been cleared by the admin (default: true). - block_auto_created_users: true - ``` + # Locks down those users until they have been cleared by the admin (default: true). + block_auto_created_users: true + ``` Now we can choose one or more of the [Supported Providers](#supported-providers) listed above to continue the configuration process. @@ -161,14 +161,14 @@ want their accounts to be upgraded to full internal accounts. **For Omnibus installations** ```ruby - gitlab_rails['omniauth_external_providers'] = ['twitter', 'google_oauth2'] +gitlab_rails['omniauth_external_providers'] = ['twitter', 'google_oauth2'] ``` **For installations from source** ```yaml - omniauth: - external_providers: ['twitter', 'google_oauth2'] +omniauth: + external_providers: ['twitter', 'google_oauth2'] ``` ## Using Custom Omniauth Providers @@ -186,23 +186,31 @@ these cases you can use the Omniauth provider. These steps are fairly general and you will need to figure out the exact details from the Omniauth provider's documentation. -- Stop GitLab: +- Stop GitLab: - sudo service gitlab stop + ```sh + sudo service gitlab stop + ``` -- Add the gem to your [Gemfile](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Gemfile): +- Add the gem to your [Gemfile](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Gemfile): - gem "omniauth-your-auth-provider" + ```sh + gem "omniauth-your-auth-provider" + ``` -- Install the new Omniauth provider gem by running the following command: +- Install the new Omniauth provider gem by running the following command: - sudo -u git -H bundle install --without development test mysql --path vendor/bundle --no-deployment + ```sh + sudo -u git -H bundle install --without development test mysql --path vendor/bundle --no-deployment + ``` - > These are the same commands you used during initial installation in the [Install Gems section](../install/installation.md#install-gems) with `--path vendor/bundle --no-deployment` instead of `--deployment`. + > These are the same commands you used during initial installation in the [Install Gems section](../install/installation.md#install-gems) with `--path vendor/bundle --no-deployment` instead of `--deployment`. -- Start GitLab: +- Start GitLab: - sudo service gitlab start + ```sh + sudo service gitlab start + ``` ### Examples @@ -247,8 +255,8 @@ gitlab_rails['omniauth_enabled'] = false **For installations from source** ```yaml - omniauth: - enabled: false +omniauth: + enabled: false ``` ## Keep OmniAuth user profiles up to date @@ -258,14 +266,14 @@ You can enable profile syncing from selected OmniAuth providers and for all or f When authenticating using LDAP, the user's name and email are always synced. ```ruby - gitlab_rails['sync_profile_from_provider'] = ['twitter', 'google_oauth2'] - gitlab_rails['sync_profile_attributes'] = ['name', 'email', 'location'] - ``` +gitlab_rails['sync_profile_from_provider'] = ['twitter', 'google_oauth2'] +gitlab_rails['sync_profile_attributes'] = ['name', 'email', 'location'] +``` **For installations from source** ```yaml - omniauth: - sync_profile_from_provider: ['twitter', 'google_oauth2'] - sync_profile_attributes: ['email', 'location'] +omniauth: + sync_profile_from_provider: ['twitter', 'google_oauth2'] + sync_profile_attributes: ['email', 'location'] ``` diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 1ef43cfcece..176622e8050 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -7,73 +7,77 @@ You can integrate your GitLab instance with [Salesforce](https://www.salesforce. To enable Salesforce OmniAuth provider, you must use Salesforce's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must [create a Connected App](https://help.salesforce.com/articleView?id=connected_app_create.htm&type=5) on Salesforce. -1. Sign in to [Salesforce](https://login.salesforce.com/). +1. Sign in to [Salesforce](https://login.salesforce.com/). -1. In Setup, enter `App Manager` in the Quick Find box, click **App Manager**, then click **New Connected App**. +1. In Setup, enter `App Manager` in the Quick Find box, click **App Manager**, then click **New Connected App**. -1. Fill in the application details into the following fields: - - **Connected App Name** and **API Name**: Set to any value but consider something like `<Organization>'s GitLab`, `<Your Name>'s GitLab`, or something else that is descriptive. - - **Contact Email**: Enter the contact email for Salesforce to use when contacting you or your support team. - - **Description**: Description for the application. +1. Fill in the application details into the following fields: + - **Connected App Name** and **API Name**: Set to any value but consider something like `<Organization>'s GitLab`, `<Your Name>'s GitLab`, or something else that is descriptive. + - **Contact Email**: Enter the contact email for Salesforce to use when contacting you or your support team. + - **Description**: Description for the application. -  -1. Select **API (Enable OAuth Settings)** and click on **Enable OAuth Settings**. -1. Fill in the application details into the following fields: - - **Callback URL**: The callback URL of your GitLab installation. For example, `https://gitlab.example.com/users/auth/salesforce/callback`. - - **Selected OAuth Scopes**: Move **Access your basic information (id, profile, email, address, phone)** and **Allow access to your unique identifier (openid)** to the right column. +  + +1. Select **API (Enable OAuth Settings)** and click on **Enable OAuth Settings**. +1. Fill in the application details into the following fields: + - **Callback URL**: The callback URL of your GitLab installation. For example, `https://gitlab.example.com/users/auth/salesforce/callback`. + - **Selected OAuth Scopes**: Move **Access your basic information (id, profile, email, address, phone)** and **Allow access to your unique identifier (openid)** to the right column. + +  -  1. Click **Save**. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. + + For omnibus package: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For omnibus package: + For installations from source: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` - For installations from source: +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` +1. Add the provider configuration: -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. + For omnibus package: -1. Add the provider configuration: + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "salesforce", + "app_id" => "SALESFORCE_CLIENT_ID", + "app_secret" => "SALESFORCE_CLIENT_SECRET" + } + ] + ``` - For omnibus package: + For installation from source: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "salesforce", - "app_id" => "SALESFORCE_CLIENT_ID", - "app_secret" => "SALESFORCE_CLIENT_SECRET" - } - ] - ``` + ``` + - { name: 'salesforce', + app_id: 'SALESFORCE_CLIENT_ID', + app_secret: 'SALESFORCE_CLIENT_SECRET' + } + ``` - For installation from source: +1. Change `SALESFORCE_CLIENT_ID` to the Consumer Key from the Salesforce connected application page. +1. Change `SALESFORCE_CLIENT_SECRET` to the Consumer Secret from the Salesforce connected application page. - ``` - - { name: 'salesforce', - app_id: 'SALESFORCE_CLIENT_ID', - app_secret: 'SALESFORCE_CLIENT_SECRET' - } - ``` -1. Change `SALESFORCE_CLIENT_ID` to the Consumer Key from the Salesforce connected application page. -1. Change `SALESFORCE_CLIENT_SECRET` to the Consumer Secret from the Salesforce connected application page. -  +  -1. Save the configuration file. -1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. +1. Save the configuration file. +1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be a Salesforce icon below the regular sign in form. Click the icon to begin the authentication process. Salesforce will ask the user to sign in and authorize the GitLab application. If everything goes well, the user will be returned to GitLab and will be signed in. NOTE: **Note:** -GitLab requires the email address of each new user. Once the user is logged in using Salesforce, GitLab will redirect the user to the profile page where they will have to provide the email and verify the email.
\ No newline at end of file +GitLab requires the email address of each new user. Once the user is logged in using Salesforce, GitLab will redirect the user to the profile page where they will have to provide the email and verify the email. diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md index 07c83c1a049..27355d25266 100644 --- a/doc/integration/shibboleth.md +++ b/doc/integration/shibboleth.md @@ -14,35 +14,35 @@ The following changes are needed to enable Shibboleth: 1. Protect omniauth-shibboleth callback URL: - ``` - <Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - require valid-user - </Location> - - Alias /shibboleth-sp /usr/share/shibboleth - <Location /shibboleth-sp> - Satisfy any - </Location> - - <Location /Shibboleth.sso> - SetHandler shib - </Location> - ``` + ``` + <Location /users/auth/shibboleth/callback> + AuthType shibboleth + ShibRequestSetting requireSession 1 + ShibUseHeaders On + require valid-user + </Location> + + Alias /shibboleth-sp /usr/share/shibboleth + <Location /shibboleth-sp> + Satisfy any + </Location> + + <Location /Shibboleth.sso> + SetHandler shib + </Location> + ``` 1. Exclude shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Config should look like this: - ``` - # Apache equivalent of Nginx try files - RewriteEngine on - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8080%{REQUEST_URI} [P,QSA] - RequestHeader set X_FORWARDED_PROTO 'https' - ``` + ``` + # Apache equivalent of Nginx try files + RewriteEngine on + RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_URI} !/Shibboleth.sso + RewriteCond %{REQUEST_URI} !/shibboleth-sp + RewriteRule .* http://127.0.0.1:8080%{REQUEST_URI} [P,QSA] + RequestHeader set X_FORWARDED_PROTO 'https' + ``` 1. Edit `/etc/gitlab/gitlab.rb` configuration file to enable OmniAuth and add Shibboleth as an OmniAuth provider. User attributes will be sent from the @@ -60,31 +60,31 @@ The following changes are needed to enable Shibboleth: The file should look like this: - ``` - external_url 'https://gitlab.example.com' - gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - - # disable Nginx - nginx['enable'] = false - - gitlab_rails['omniauth_allow_single_sign_on'] = true - gitlab_rails['omniauth_block_auto_created_users'] = false - gitlab_rails['omniauth_enabled'] = true - gitlab_rails['omniauth_providers'] = [ - { - "name" => "'shibboleth"', - "label" => "Text for Login Button", - "args" => { - "shib_session_id_field" => "HTTP_SHIB_SESSION_ID", - "shib_application_id_field" => "HTTP_SHIB_APPLICATION_ID", - "uid_field" => 'HTTP_EPPN', - "name_field" => 'HTTP_CN', - "info_fields" => { "email" => 'HTTP_MAIL'} - } - } - ] - - ``` + ``` + external_url 'https://gitlab.example.com' + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # disable Nginx + nginx['enable'] = false + + gitlab_rails['omniauth_allow_single_sign_on'] = true + gitlab_rails['omniauth_block_auto_created_users'] = false + gitlab_rails['omniauth_enabled'] = true + gitlab_rails['omniauth_providers'] = [ + { + "name" => "'shibboleth"', + "label" => "Text for Login Button", + "args" => { + "shib_session_id_field" => "HTTP_SHIB_SESSION_ID", + "shib_application_id_field" => "HTTP_SHIB_APPLICATION_ID", + "uid_field" => 'HTTP_EPPN', + "name_field" => 'HTTP_CN', + "info_fields" => { "email" => 'HTTP_MAIL'} + } + } + ] + + ``` 1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../administration/restart_gitlab.md#installations-from-source) GitLab for the changes to take effect if you installed GitLab via Omnibus or from source respectively. @@ -97,44 +97,44 @@ The order of the first 2 Location directives is important. If they are reversed, you will not get a shibboleth session! ``` - <Location /> - Require all granted - ProxyPassReverse http://127.0.0.1:8181 - ProxyPassReverse http://YOUR_SERVER_FQDN/ - </Location> - - <Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - Require shib-session - </Location> - - Alias /shibboleth-sp /usr/share/shibboleth - - <Location /shibboleth-sp> - Require all granted - </Location> - - <Location /Shibboleth.sso> - SetHandler shib - </Location> - - RewriteEngine on - - #Don't escape encoded characters in api requests - RewriteCond %{REQUEST_URI} ^/api/v4/.* - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA,NE] - - #Forward all requests to gitlab-workhorse except existing files - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f [OR] - RewriteCond %{REQUEST_URI} ^/uploads/.* - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA] - - RequestHeader set X_FORWARDED_PROTO 'https' - RequestHeader set X-Forwarded-Ssl on +<Location /> + Require all granted + ProxyPassReverse http://127.0.0.1:8181 + ProxyPassReverse http://YOUR_SERVER_FQDN/ +</Location> + +<Location /users/auth/shibboleth/callback> + AuthType shibboleth + ShibRequestSetting requireSession 1 + ShibUseHeaders On + Require shib-session +</Location> + +Alias /shibboleth-sp /usr/share/shibboleth + +<Location /shibboleth-sp> + Require all granted +</Location> + +<Location /Shibboleth.sso> + SetHandler shib +</Location> + +RewriteEngine on + +#Don't escape encoded characters in api requests +RewriteCond %{REQUEST_URI} ^/api/v4/.* +RewriteCond %{REQUEST_URI} !/Shibboleth.sso +RewriteCond %{REQUEST_URI} !/shibboleth-sp +RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA,NE] + +#Forward all requests to gitlab-workhorse except existing files +RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f [OR] +RewriteCond %{REQUEST_URI} ^/uploads/.* +RewriteCond %{REQUEST_URI} !/Shibboleth.sso +RewriteCond %{REQUEST_URI} !/shibboleth-sp +RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA] + +RequestHeader set X_FORWARDED_PROTO 'https' +RequestHeader set X-Forwarded-Ssl on ``` diff --git a/doc/integration/ultra_auth.md b/doc/integration/ultra_auth.md index 69b2a75050d..9ed1bdb4882 100644 --- a/doc/integration/ultra_auth.md +++ b/doc/integration/ultra_auth.md @@ -7,69 +7,78 @@ You can integrate your GitLab instance with [UltraAuth](https://ultraauth.com) t To enable UltraAuth OmniAuth provider, you must use UltraAuth's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must register an application on UltraAuth. -1. Sign in to [UltraAuth](https://ultraauth.com). -1. Navigate to [Create an App](https://ultraauth.com/select-strategy) and click on "Ruby on Rails". -1. Scroll down the page that is displayed to locate the **Client ID** and **Client Secret**. - Keep this page open as you continue configuration. -  -1. Click on "Edit Callback URL" link. -  -1. The callback URL will be `http(s)://<your_domain>/users/auth/ultraauth/callback` -  -1. Select **Register application**. -1. On your GitLab server, open the configuration file. - - For omnibus package: - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: - - For omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "ultraauth", - "app_id" => "OPENID_CLIENT_ID", - "app_secret" => "OPENID_CLIENT_SECRET", - "args" => { - "client_options" => { - "redirect_uri" => "https://example.com/users/auth/ultraauth/callback" - } - } - } - ] - ``` - - For installation from source: - - ``` - - { name: 'ultraauth', - app_id: 'OPENID_CLIENT_ID', - app_secret: 'OPENID_CLIENT_SECRET', - args: { - client_options: { - redirect_uri: 'https://example.com/users/auth/ultraauth/callback' - } - } - } - ``` - __Replace `https://example.com/users/auth/ultraauth/callback` with your application's Callback URL.__ -1. Change `OPENID_CLIENT_ID` to the Client ID from the UltraAuth application page. -1. Change `OPENID_CLIENT_SECRET` to the Client Secret from the UltraAuth application page. -1. Save the configuration file. -1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Sign in to [UltraAuth](https://ultraauth.com). +1. Navigate to [Create an App](https://ultraauth.com/select-strategy) and click on "Ruby on Rails". +1. Scroll down the page that is displayed to locate the **Client ID** and **Client Secret**. + Keep this page open as you continue configuration. + +  + +1. Click on "Edit Callback URL" link. + +  + +1. The callback URL will be `http(s)://<your_domain>/users/auth/ultraauth/callback` + +  + +1. Select **Register application**. +1. On your GitLab server, open the configuration file. + + For omnibus package: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. Add the provider configuration: + + For omnibus package: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "ultraauth", + "app_id" => "OPENID_CLIENT_ID", + "app_secret" => "OPENID_CLIENT_SECRET", + "args" => { + "client_options" => { + "redirect_uri" => "https://example.com/users/auth/ultraauth/callback" + } + } + } + ] + ``` + + For installation from source: + + ``` + - { name: 'ultraauth', + app_id: 'OPENID_CLIENT_ID', + app_secret: 'OPENID_CLIENT_SECRET', + args: { + client_options: { + redirect_uri: 'https://example.com/users/auth/ultraauth/callback' + } + } + } + ``` + + __Replace `https://example.com/users/auth/ultraauth/callback` with your application's Callback URL.__ + +1. Change `OPENID_CLIENT_ID` to the Client ID from the UltraAuth application page. +1. Change `OPENID_CLIENT_SECRET` to the Client Secret from the UltraAuth application page. +1. Save the configuration file. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be an UltraAuth icon below the regular sign in form. Click the icon to begin the authentication process. UltraAuth will ask the user to sign in and authorize the GitLab application. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 092b4375208..9b1a4105dc3 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -741,9 +741,10 @@ sudo gitlab-rake gitlab:backup:restore BACKUP=1493107454_2018_04_25_10.6.4-ce Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary as mentioned above. -Restart and check GitLab: +Reconfigure, restart and check GitLab: ```shell +sudo gitlab-ctl reconfigure sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index b59c06a24ea..8f65fab366e 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -100,7 +100,7 @@ the git repository's config file. This section is formatted as follows: ``` [gitlab] - fullpath = gitlab-org/gitlab-ce + fullpath = gitlab-org/gitlab-ce ``` However, existing repositories were not migrated to include this path. diff --git a/doc/system_hooks/system_hooks.md b/doc/system_hooks/system_hooks.md index a46f7d30892..e8bd35fba5c 100644 --- a/doc/system_hooks/system_hooks.md +++ b/doc/system_hooks/system_hooks.md @@ -330,6 +330,7 @@ If the user is blocked via LDAP, `state` will be `ldap_blocked`. "user_id": 41 } ``` + **Group Member Removed:** ```json diff --git a/doc/university/training/topics/env_setup.md b/doc/university/training/topics/env_setup.md index 305f5ecb1fb..92d2613c5d2 100644 --- a/doc/university/training/topics/env_setup.md +++ b/doc/university/training/topics/env_setup.md @@ -14,9 +14,11 @@ comments: false - If it's not installed, it will prompt you to install it. - **Linux** + ```bash sudo yum install git-all ``` + ```bash sudo apt-get install git-all ``` diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index 9e7057f93d4..39753fd885e 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -64,8 +64,8 @@ Internal URL defaults to External URL, but you can customize it under CAUTION: **Warning:** We recommend using an HTTPS connection while configuring the Geo nodes. To avoid breaking communication between **primary** and **secondary** nodes when using -HTTPS, customize your Internal URL to point to a Load Balancer with TLS -termination. +HTTPS, customize your Internal URL to point to a load balancer with TLS +terminated at the load balancer. ## Multiple secondary nodes behind a load balancer diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index 06cc64b209d..df86b2a1cbe 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -170,6 +170,7 @@ Attach a block or paragraph to a list item using a list continuation (which you * [x] checked * [ ] not checked ``` + #### Callout ```asciidoc @@ -188,6 +189,7 @@ first term:: description of first term second term:: description of second term ``` + ### Document Structure #### Header @@ -197,6 +199,7 @@ description of second term Author Name <author@example.org> v1.0, 2019-01-01 ``` + #### Sections ```asciidoc @@ -217,6 +220,7 @@ include::basics.adoc[] // define -a allow-uri-read to allow content to be read from URI include::https://example.org/installation.adoc[] ``` + ### Blocks ```asciidoc @@ -273,11 +277,11 @@ source - a listing that is embellished with (colorized) syntax highlighting ---- ``` -```asciidoc +````asciidoc \```language fenced code - a shorthand syntax for the source block \``` -``` +```` ```asciidoc [,attribution,citetitle] @@ -369,4 +373,3 @@ video::300817511[vimeo] // page break <<< ``` - diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index 8a85c375490..7cdba8cf2b5 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -24,7 +24,7 @@ project in the group will be available to every logged in user. However, private projects will be available only if the user is a member of the project. NOTE: **Note:** -Projects of nested subgroups of a selected template source cannot be used. +Only direct subgroups can be set as the template source. Projects of nested subgroups of a selected template source cannot be used. Repository and database information that are copied over to each new project are identical to the data exported with [GitLab's Project Import/Export](../project/settings/import_export.md). diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 83629fc2e4b..0d3bbeff4e5 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -40,7 +40,7 @@ repositories are also processed with CommonMark. As of 11.8, the [Redcarpet Ruby has been removed and all issues and comments, including those from pre-11.1, are now processed using the [CommonMark Ruby Library](https://github.com/gjtorikian/commonmarker). -The documentation website had its [markdown engine migrated from Redcarpet to Kramdown](https://gitlab.com/gitlab-com/gitlab-docs/merge_requests/108) +The documentation website had its [markdown engine migrated from Redcarpet to Kramdown](https://gitlab.com/gitlab-org/gitlab-docs/merge_requests/108) in October 2018. You may have older issues, merge requests, or Markdown documents in your diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index a06c3d3c662..a8473f76733 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -102,12 +102,15 @@ You must do the following: 1. Ensure GitLab can manage Knative: - For a non-GitLab managed cluster, ensure that the service account for the token provided can manage resources in the `serving.knative.dev` API group. - - For a GitLab managed cluster, - GitLab uses a service account with the `edit` cluster role. This account needs - the ability to manage resources in the `serving.knative.dev` API group. - We suggest you do this with an [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) - adding rules to the default `edit` cluster role: - First, save the following YAML as `knative-serving-only-role.yaml`: + - For a GitLab managed cluster, if you added the cluster in [GitLab 12.1 or later](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30235), + then GitLab will already have the required access and you can proceed to the next step. + + Otherwise, you need to manually grant GitLab's service account the ability to manage + resources in the `serving.knative.dev` API group. Since every GitLab service account + has the `edit` cluster role, the simplest way to do this is with an + [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) + adding rules to the default `edit` cluster role: First, save the following YAML as + `knative-serving-only-role.yaml`: ```yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -143,6 +146,9 @@ You must do the following: kubectl apply -f knative-serving-only-role.yaml ``` + If you would rather grant permissions on a per service account basis, you can do this + using a `Role` and `RoleBinding` specific to the service account and namespace. + 1. Follow the steps to deploy [functions](#deploying-functions) or [serverless applications](#deploying-serverless-applications) onto your cluster. @@ -376,13 +382,13 @@ cluster. By default, a GitLab serverless deployment will be served over `http`. In order to serve over `https` you must manually obtain and install TLS certificates. -The simplest way to accomplish this is to +The simplest way to accomplish this is to use [Certbot to manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). Certbot is a free, open source software tool for automatically using Let’s Encrypt certificates on manually-administrated websites to enable HTTPS. NOTE: **Note:** The instructions below relate to installing and running Certbot on a Linux server and may not work on other operating systems. -1. Install Certbot by running the +1. Install Certbot by running the [`certbot-auto` wrapper script](https://certbot.eff.org/docs/install.html#certbot-auto). On the command line of your server, run the following commands: @@ -594,7 +600,7 @@ The instructions below relate to installing and running Certbot on a Linux serve Where `cert.pem` and `cert.pk` are your certificate and private key files. Note that the `istio-ingressgateway-certs` secret name is required. 1. Configure Knative to use the new secret that you created for HTTPS - connections. Run the + connections. Run the following command to open the Knative shared `gateway` in edit mode: ```sh @@ -641,4 +647,4 @@ The instructions below relate to installing and running Certbot on a Linux serve After your changes are running on your Knative cluster, you can begin using the HTTPS protocol for secure access your deployed Knative services. In the event a mistake is made during this process and you need to update the cert, you will need to edit the gateway `knative-ingress-gateway` - to switch back to `PASSTHROUGH` mode. Once corrections are made, edit the file again so the gateway will use the new certificates.
\ No newline at end of file + to switch back to `PASSTHROUGH` mode. Once corrections are made, edit the file again so the gateway will use the new certificates. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 78ffa11d59b..96c4f16fe04 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -57,8 +57,8 @@ Example `CODEOWNERS` file: # Files with a `#` can still be accesssed by escaping the pound sign \#file_with_pound.rb @owner-file-with-pound -# Multiple codeowners can be specified, separated by whitespace -CODEOWNERS @multiple @owners @tab-separated +# Multiple codeowners can be specified, separated by spaces or tabs +CODEOWNERS @multiple @code @owners # Both usernames or email addresses can be used to match # users. Everything else will be ignored. For example this will diff --git a/doc/user/project/import/phabricator.md b/doc/user/project/import/phabricator.md index 5c624e3aff6..b8f89caba24 100644 --- a/doc/user/project/import/phabricator.md +++ b/doc/user/project/import/phabricator.md @@ -15,6 +15,12 @@ Currently, only the following basic fields are imported: - Created at - Closed at +## Users + +The assignee and author of a user are deducted from a Task's owner and +author: If a user with the same username has access to the namespace +of the project being imported into, then the user will be linked. + ## Enabling this feature While this feature is incomplete, a feature flag is required to enable it so that diff --git a/doc/user/project/integrations/project_services.md b/doc/user/project/integrations/project_services.md index 62e08a183f7..f63a2c2758b 100644 --- a/doc/user/project/integrations/project_services.md +++ b/doc/user/project/integrations/project_services.md @@ -20,10 +20,10 @@ Below, you will find a list of the currently supported ones accompanied with com Click on the service links to see further configuration instructions and details. -| Service | Description | +| Service | Description | | ------- | ----------- | -| Asana | Asana - Teamwork without email | -| Assembla | Project Management Software (Source Commits Endpoint) | +| Asana | Asana - Teamwork without email | +| Assembla | Project Management Software (Source Commits Endpoint) | | [Atlassian Bamboo CI](bamboo.md) | A continuous integration and build server | | Buildkite | Continuous integration and deployments | | [Bugzilla](bugzilla.md) | Bugzilla issue tracker | diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 01b6650bfab..765aa91b00f 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -121,7 +121,70 @@ GitLab supports a limited set of [CI variables](../../../ci/variables/README.htm To specify a variable in a query, enclose it in curly braces with a leading percent. For example: `%{ci_environment_slug}`. -### Setting up alerts for Prometheus metrics **(ULTIMATE)** +### Defining Dashboards for Prometheus Metrics per Project + +All projects include a GitLab-defined system dashboard, which includes a few key metrics. Optionally, additional dashboards can also be defined by including configuration files in the project repository under `.gitlab/dashboards`. Configuration files nested under subdirectories will not be available in the UI. Each file should define the layout of the dashboard and the prometheus queries used to populate data. Dashboards can be selected from the dropdown in the UI. + +#### Relationship to Custom Metrics + +[Custom Metrics](#adding-additional-metrics-premium) are defined through the UI and, at this point, are unique from metrics defined in dashboard configuration files. Custom Metrics will appear on the system dashboard, as well as support alerting, whereas metrics defined in configuration files do not yet support alerts. + +#### Dashboard Configuration + +Dashboards have several components. A dashboard has many panel groups, which are comprised of panels, which support one or more metrics. The dashboard should be saved with the `.yml` extension. + +Sample YML Configuration +``` +dashboard: 'Dashboard Title' +priority: 2 +panel_groups: + - group: 'Group Title' + panels: + - type: area-chart + title: "Chart Title" + y_label: "Y-Axis" + metrics: + - id: metric_of_ages + query_range: 'http_requests_total' + label: "Metric of Ages" + unit: "count" +``` + +The above sample dashboard would display a single area chart. The following sections outline the details of expected properties. + +##### Dashboard Properties +| Property | Type | Required? | Meaning | +| ------ | ------ | ------ | ------ | +| `dashboard` | string | required | Heading for the dashboard. Only one dashboard should be defined per file. | +| `priority` | number | optional, default to definition order | Order to appear in dashboard dropdown, higher priority should be higher in the dropdown. Numbers do not need to be consecutive. | +| `panel_groups` | array | required | The panel groups which should be on the dashboard. | + +##### Panel Group Properties +| Property | Type | Required? | Meaning | +| ------ | ------ | ------ | ------ | +| `group` | string | required | Heading for the panel group. | +| `priority` | number | optional, defaults to order in file | Order to appear on the dashboard, higher priority will be higher on the page. Numbers do not need to be consecutive. | +| `panels` | array | required | The panels which should be in the panel group. | + +##### Panel Properties +| Property | Type | Required? | Meaning | +| ------ | ------ | ------ | ------- | +| `type` | enum | optional, defaults to `area-chart` | Specifies the chart type to use. Only `area-chart` is currently supported. | +| `title` | string | required | Heading for the panel. | +| `y_label` | string | optional, but highly encouraged | Y-Axis label for the panel. | +| `weight` | number | optional, defaults to order in file | Order to appear within the grouping, higher priority will be higher on the page. Numbers do not need to be consecutive. | +| `metrics` | array | required | The metrics which should be displayed in the panel. | + +##### Metric Properties +| Property | Type | Required? | Meaning | +| ------ | ------ | ------ | ------ | +| `id` | string | optional | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](#setting-up-alerts-for-prometheus-metrics-ultimate) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/60319)). | +| `unit` | string | required | Defines the unit of the query's return data. | +| `label` | string | optional, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. | +| `query` | string | required unless `query_range` is defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +| `query_range` | string | required unless `query` is defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | + +### Setting up alerts for Prometheus metrics **[ULTIMATE]** #### Managed Prometheus instances diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 37a0630d0f3..c299a8ecb96 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -274,7 +274,8 @@ branch already exists, the patches will be applied on top of it. > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26752) in GitLab 11.10. NOTE: **Note:** -Git push options are only available with Git 2.10 or newer. +Git push options are only available with Git 2.10 or newer. With Git older than 2.18 +`git push --push-option=...` should be used instead of `git push -o ...`. GitLab supports using [Git push options](https://git-scm.com/docs/git-push#Documentation/git-push.txt--oltoptiongt) diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index 03af8fad759..55183408a10 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -148,20 +148,20 @@ On Omnibus installations, the settings are prefixed by `lfs_object_store_`: 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with the values you want: - ```ruby - gitlab_rails['lfs_object_store_enabled'] = true - gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" - gitlab_rails['lfs_object_store_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', - 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', - # The below options configure an S3 compatible host instead of AWS - 'host' => 'localhost', - 'endpoint' => 'http://127.0.0.1:9000', - 'path_style' => true - } - ``` + ```ruby + gitlab_rails['lfs_object_store_enabled'] = true + gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" + gitlab_rails['lfs_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', + 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', + # The below options configure an S3 compatible host instead of AWS + 'host' => 'localhost', + 'endpoint' => 'http://127.0.0.1:9000', + 'path_style' => true + } + ``` 1. Save the file and [reconfigure GitLab]s for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: @@ -182,22 +182,22 @@ For source installations the settings are nested under `lfs:` and then 1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: - ```yaml - lfs: - enabled: true - object_store: - enabled: false - remote_directory: lfs-objects # Bucket name - connection: - provider: AWS - aws_access_key_id: 1ABCD2EFGHI34JKLM567N - aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE - region: eu-central-1 - # Use the following options to configure an AWS compatible host such as Minio - host: 'localhost' - endpoint: 'http://127.0.0.1:9000' - path_style: true - ``` + ```yaml + lfs: + enabled: true + object_store: + enabled: false + remote_directory: lfs-objects # Bucket name + connection: + provider: AWS + aws_access_key_id: 1ABCD2EFGHI34JKLM567N + aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE + region: eu-central-1 + # Use the following options to configure an AWS compatible host such as Minio + host: 'localhost' + endpoint: 'http://127.0.0.1:9000' + path_style: true + ``` 1. Save the file and [restart GitLab][] for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: diff --git a/jest.config.js b/jest.config.js index 986b8465eef..e4ac71a1a17 100644 --- a/jest.config.js +++ b/jest.config.js @@ -19,7 +19,7 @@ let testMatch = ['<rootDir>/spec/frontend/**/*_spec.js', '<rootDir>/ee/spec/fron // workaround for eslint-import-resolver-jest only resolving in test files // see https://github.com/JoinColony/eslint-import-resolver-jest#note -const isESLint = module.parent.path.includes('/eslint-import-resolver-jest/'); +const isESLint = module.parent.filename.includes('/eslint-import-resolver-jest/'); if (isESLint) { testMatch = testMatch.map(path => path.replace('_spec.js', '')); } diff --git a/lib/api/api.rb b/lib/api/api.rb index 7016a66593d..574934d7f90 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -111,6 +111,7 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards + mount ::API::GroupClusters mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 765819e6bf1..0a9515f1dd2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1686,5 +1686,9 @@ module API class ClusterProject < Cluster expose :project, using: Entities::BasicProjectDetails end + + class ClusterGroup < Cluster + expose :group, using: Entities::BasicGroupDetails + end end end diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb new file mode 100644 index 00000000000..db0f8081140 --- /dev/null +++ b/lib/api/group_clusters.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module API + class GroupClusters < Grape::API + include PaginationParams + + before { authenticate! } + + # EE::API::GroupClusters will + # override these methods + helpers do + params :create_params_ee do + end + + params :update_params_ee do + end + end + + params do + requires :id, type: String, desc: 'The ID of the group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all clusters from the group' do + success Entities::Cluster + end + params do + use :pagination + end + get ':id/clusters' do + authorize! :read_cluster, user_group + + present paginate(clusters_for_current_user), with: Entities::Cluster + end + + desc 'Get specific cluster for the group' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + end + get ':id/clusters/:cluster_id' do + authorize! :read_cluster, cluster + + present cluster, with: Entities::ClusterGroup + end + + desc 'Adds an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' + optional :domain, type: String, desc: 'Cluster base domain' + optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' + requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' + requires :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC' + end + use :create_params_ee + end + post ':id/clusters/user' do + authorize! :add_cluster, user_group + + user_cluster = ::Clusters::CreateService + .new(current_user, create_cluster_user_params) + .execute + + if user_cluster.persisted? + present user_cluster, with: Entities::ClusterGroup + else + render_validation_error!(user_cluster) + end + end + + desc 'Update an existing cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + optional :name, type: String, desc: 'Cluster name' + optional :domain, type: String, desc: 'Cluster base domain' + optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + optional :api_url, type: String, desc: 'URL to access the Kubernetes API' + optional :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Group' + end + use :update_params_ee + end + put ':id/clusters/:cluster_id' do + authorize! :update_cluster, cluster + + update_service = Clusters::UpdateService.new(current_user, update_cluster_params) + + if update_service.execute(cluster) + present cluster, with: Entities::ClusterGroup + else + render_validation_error!(cluster) + end + end + + desc 'Remove a cluster' do + success Entities::ClusterGroup + end + params do + requires :cluster_id, type: Integer, desc: 'The Cluster ID' + end + delete ':id/clusters/:cluster_id' do + authorize! :admin_cluster, cluster + + destroy_conditionally!(cluster) + end + end + + helpers do + def clusters_for_current_user + @clusters_for_current_user ||= ClustersFinder.new(user_group, current_user, :all).execute + end + + def cluster + @cluster ||= clusters_for_current_user.find(params[:cluster_id]) + end + + def create_cluster_user_params + declared_params.merge({ + provider_type: :user, + platform_type: :kubernetes, + clusterable: user_group + }) + end + + def update_cluster_params + declared_params(include_missing: false).without(:cluster_id) + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index c318f5b9127..9afe6c5b027 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -72,7 +72,7 @@ module API result == 'PONG' rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 2a9b17ad22a..71bbc218f94 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -205,7 +205,9 @@ module API limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT - pagination_data.without_count + # The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?` + # We need to call `reset` because `without_count` relies on `@arel` being unmemoized + pagination_data.reset.without_count else pagination_data end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index ff73a49d5e8..100463fcb95 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -7,8 +7,7 @@ module API JOB_TOKEN_PARAM = :token def runner_registration_token_valid? - ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - Gitlab::CurrentSettings.runners_registration_token) + ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) end def authenticate_runner! diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index dcc8d94fb79..4f093e9be08 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -65,7 +65,7 @@ module API use :create_params_ee end post ':id/clusters/user' do - authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters' + authorize! :add_cluster, user_project user_cluster = ::Clusters::CreateService .new(current_user, create_cluster_user_params) diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb new file mode 100644 index 00000000000..a78bb60103c --- /dev/null +++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by AsciiDoc/Asciidoctor. + # + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class AsciiDocSanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Classes used by Asciidoctor to style components + ADMONITION_CLASSES = %w(fa icon-note icon-tip icon-warning icon-caution icon-important).freeze + CALLOUT_CLASSES = ['conum'].freeze + CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze + + LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze + + ELEMENT_CLASSES_WHITELIST = { + span: %w(big small underline overline line-through).freeze, + div: ['admonitionblock'].freeze, + td: ['icon'].freeze, + i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES, + ul: LIST_CLASSES, + ol: LIST_CLASSES + }.freeze + + def customize_whitelist(whitelist) + # Allow marks + whitelist[:elements].push('mark') + + # Allow any classes in `span`, `i`, `div`, `td`, `ul` and `ol` elements + # but then remove any unknown classes + whitelist[:attributes]['span'] = %w(class) + whitelist[:attributes]['div'].push('class') + whitelist[:attributes]['td'] = %w(class) + whitelist[:attributes]['i'] = %w(class) + whitelist[:attributes]['ul'] = %w(class) + whitelist[:attributes]['ol'] = %w(class) + whitelist[:transformers].push(self.class.remove_element_classes) + + whitelist + end + + class << self + def remove_element_classes + lambda do |env| + node = env[:node] + + return unless (classes_whitelist = ELEMENT_CLASSES_WHITELIST[node.name.to_sym]) + return unless node.has_attribute?('class') + + classes = node['class'].strip.split(' ') + allowed_classes = (classes & classes_whitelist) + if allowed_classes.empty? + node.remove_attribute('class') + else + node['class'] = allowed_classes.join(' ') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb new file mode 100644 index 00000000000..420e92cb1e8 --- /dev/null +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Sanitize HTML produced by markup languages (Markdown, AsciiDoc...). + # Specific rules are implemented in dedicated filters: + # + # - Banzai::Filter::SanitizationFilter (Markdown) + # - Banzai::Filter::AsciiDocSanitizationFilter (AsciiDoc/Asciidoctor) + # + # Extends HTML::Pipeline::SanitizationFilter with common rules. + class BaseSanitizationFilter < HTML::Pipeline::SanitizationFilter + include Gitlab::Utils::StrongMemoize + + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + + def whitelist + strong_memoize(:whitelist) do + whitelist = super.deep_dup + + # Allow span elements + whitelist[:elements].push('span') + + # Allow data-math-style attribute in order to support LaTeX formatting + whitelist[:attributes]['code'] = %w(data-math-style) + whitelist[:attributes]['pre'] = %w(data-math-style) + + # Allow html5 details/summary elements + whitelist[:elements].push('details') + whitelist[:elements].push('summary') + + # Allow abbr elements with title attribute + whitelist[:elements].push('abbr') + whitelist[:attributes]['abbr'] = %w(title) + + # Disallow `name` attribute globally, allow on `a` + whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') + + # Allow any protocol in `a` elements + # and then remove links with unsafe protocols + whitelist[:protocols].delete('a') + whitelist[:transformers].push(self.class.remove_unsafe_links) + + # Remove `rel` attribute from `a` elements + whitelist[:transformers].push(self.class.remove_rel) + + customize_whitelist(whitelist) + end + end + + def customize_whitelist(whitelist) + raise NotImplementedError + end + + class << self + def remove_unsafe_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + begin + node['href'] = node['href'].strip + uri = Addressable::URI.parse(node['href']) + + return unless uri.scheme + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + scheme = uri.scheme + .strip + .downcase + .gsub(/[^A-Za-z0-9\+\.\-]+/, '') + + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) + rescue Addressable::URI::InvalidURIError + node.remove_attribute('href') + end + end + end + + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end + end + end + end + end + end +end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index a4a06eae7b7..f57e57890f8 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -2,23 +2,13 @@ module Banzai module Filter - # Sanitize HTML + # Sanitize HTML produced by Markdown. # - # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. - class SanitizationFilter < HTML::Pipeline::SanitizationFilter - include Gitlab::Utils::StrongMemoize - - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. + class SanitizationFilter < Banzai::Filter::BaseSanitizationFilter + # Styles used by Markdown for table alignment TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze - def whitelist - strong_memoize(:whitelist) do - customize_whitelist(super.deep_dup) - end - end - - private - def customize_whitelist(whitelist) # Allow table alignment; we whitelist specific text-align values in a # transformer below @@ -26,36 +16,9 @@ module Banzai whitelist[:attributes]['td'] = %w(style) whitelist[:css] = { properties: ['text-align'] } - # Allow span elements - whitelist[:elements].push('span') - - # Allow data-math-style attribute in order to support LaTeX formatting - whitelist[:attributes]['code'] = %w(data-math-style) - whitelist[:attributes]['pre'] = %w(data-math-style) - - # Allow html5 details/summary elements - whitelist[:elements].push('details') - whitelist[:elements].push('summary') - - # Allow abbr elements with title attribute - whitelist[:elements].push('abbr') - whitelist[:attributes]['abbr'] = %w(title) - # Allow the 'data-sourcepos' from CommonMark on all elements whitelist[:attributes][:all].push('data-sourcepos') - # Disallow `name` attribute globally, allow on `a` - whitelist[:attributes][:all].delete('name') - whitelist[:attributes]['a'].push('name') - - # Allow any protocol in `a` elements - # and then remove links with unsafe protocols - whitelist[:protocols].delete('a') - whitelist[:transformers].push(self.class.remove_unsafe_links) - - # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(self.class.remove_rel) - # Remove any `style` properties not required for table alignment whitelist[:transformers].push(self.class.remove_unsafe_table_style) @@ -69,43 +32,6 @@ module Banzai end class << self - def remove_unsafe_links - lambda do |env| - node = env[:node] - - return unless node.name == 'a' - return unless node.has_attribute?('href') - - begin - node['href'] = node['href'].strip - uri = Addressable::URI.parse(node['href']) - - return unless uri.scheme - - # Remove all invalid scheme characters before checking against the - # list of unsafe protocols. - # - # See https://tools.ietf.org/html/rfc3986#section-3.1 - scheme = uri.scheme - .strip - .downcase - .gsub(/[^A-Za-z0-9\+\.\-]+/, '') - - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') - end - end - end - - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') - end - end - end - def remove_unsafe_table_style lambda do |env| node = env[:node] diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 6be489c6572..d25b74b23b2 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -5,7 +5,7 @@ module Banzai class AsciiDocPipeline < BasePipeline def self.filters FilterArray[ - Filter::SanitizationFilter, + Filter::AsciiDocSanitizationFilter, Filter::SyntaxHighlightFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7239a5eaa6..81f32ef5bcf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -55,11 +55,16 @@ module Banzai # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # - # As the rendered Markdown String can be already cached read all the data - # from the cache using Rails.cache.read_multi operation. If the Markdown String - # is not in the cache or it's not cacheable (no cache_key entry is provided in - # the context) the Markdown String is rendered and stored in the cache so the - # next render call gets the rendered HTML-safe String from the cache. + # The redis cache is completely obviated if we receive a `:rendered` key in the + # context, as it is assumed the item has been pre-rendered somewhere else and there + # is no need to cache it. + # + # If no `:rendered` key is present in the context, as the rendered Markdown String + # can be already cached, read all the data from the cache using + # Rails.cache.read_multi operation. If the Markdown String is not in the cache + # or it's not cacheable (no cache_key entry is provided in the context) the + # Markdown String is rendered and stored in the cache so the next render call + # gets the rendered HTML-safe String from the cache. # # For further explanation see #render method comments. # @@ -76,19 +81,34 @@ module Banzai # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] def self.cache_collection_render(texts_and_contexts) - items_collection = texts_and_contexts.each_with_index do |item, index| + items_collection = texts_and_contexts.each do |item| context = item[:context] - cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) - item[:cache_key] = cache_key if cache_key + if context.key?(:rendered) + item[:rendered] = context.delete(:rendered) + else + # If the attribute didn't come in pre-rendered, let's prepare it for caching it in redis + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + item[:cache_key] = cache_key if cache_key + end end - cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + cacheable_items, non_cacheable_items = items_collection.group_by do |item| + if item.key?(:rendered) + # We're not really doing anything here as these don't need any processing, but leaving it just in case + # as they could have a cache_key and we don't want them to be re-rendered + :rendered + elsif item.key?(:cache_key) + :cacheable + else + :non_cacheable + end + end.values_at(:cacheable, :non_cacheable) items_in_cache = [] items_not_in_cache = [] - unless cacheable_items.empty? + if cacheable_items.present? items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) items_not_in_cache = cacheable_items.reject do |item| item[:rendered] = items_in_cache[item[:cache_key]] @@ -96,7 +116,7 @@ module Banzai end end - (items_not_in_cache + non_cacheable_items).each do |item| + (items_not_in_cache + Array.wrap(non_cacheable_items)).each do |item| item[:rendered] = render(item[:text], item[:context]) Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 4317992d933..82e0c7ceeaa 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -94,6 +94,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def rate_limit!(ip, success:, login:) rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) return unless rate_limiter.enabled? @@ -114,6 +115,7 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 15b9d5ad6e9..bcb0ecccdf9 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -55,7 +55,7 @@ module Gitlab response = ldap.get_operation_result unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") + Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger end [] @@ -67,7 +67,7 @@ module Gitlab retries += 1 error_message = connection_error_message(error) - Rails.logger.warn(error_message) + Rails.logger.warn(error_message) # rubocop:disable Gitlab/RailsLogger if retries < MAX_SEARCH_RETRIES renew_connection_adapter diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 47d63eb53cf..354f91306f9 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -240,7 +240,7 @@ module Gitlab begin custom_options[:cert] = OpenSSL::X509::Certificate.new(custom_options[:cert]) rescue OpenSSL::X509::CertificateError => e - Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'cert' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end @@ -248,7 +248,7 @@ module Gitlab begin custom_options[:key] = OpenSSL::PKey.read(custom_options[:key]) rescue OpenSSL::PKey::PKeyError => e - Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" + Rails.logger.error "LDAP TLS Options 'key' is invalid for provider #{provider}: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index c1517222956..11a4052a109 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -45,7 +45,7 @@ module Gitlab def self.normalize_dn(dn) ::Gitlab::Auth::LDAP::DN.new(dn).to_normalized_s rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger dn end @@ -57,13 +57,13 @@ module Gitlab def self.normalize_uid(uid) ::Gitlab::Auth::LDAP::DN.normalize_value(uid) rescue ::Gitlab::Auth::LDAP::DN::FormatError => e - Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") # rubocop:disable Gitlab/RailsLogger uid end def initialize(entry, provider) - Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } + Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } # rubocop:disable Gitlab/RailsLogger @entry = entry @provider = provider end diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index cb2bdea755c..c912628d0fc 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -9,7 +9,7 @@ module Gitlab end def perform(start_id, stop_id) - Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger update = ' commits_count = ( diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb index 7ee783b8489..3c26982729d 100644 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -14,7 +14,7 @@ module Gitlab build.trace.archive! rescue => e - Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 886c41a2b9d..e62f5edd0e7 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -10,7 +10,7 @@ module Gitlab .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) rescue => e - Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 0e93b2cb2fa..6046d33aeac 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -32,7 +32,7 @@ module Gitlab ) end rescue => e - Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" + Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb index bf5d7f5f322..20a98c8e141 100644 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -108,7 +108,7 @@ module Gitlab next unless matching_label - Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" + Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger LabelLink.update(label.label_link_id, label_id: matching_label.id) end end diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb new file mode 100644 index 00000000000..1a207121be0 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the namespaces.name for all user-namespaces that have names + # that aren't equal to the users name. + # Then it uses the updated names of the namespaces to update the associated routes + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserNamespaceNames + def perform(from_id, to_id) + fix_namespace_names(from_id, to_id) + fix_namespace_route_names(from_id, to_id) + end + + def fix_namespace_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES + WITH namespaces_to_update AS ( + SELECT + namespaces.id, + users.name AS correct_name + FROM + namespaces + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.name != users.name + ) + UPDATE + namespaces + SET + name = correct_name + FROM + namespaces_to_update + WHERE + namespaces.id = namespaces_to_update.id + UPDATE_NAMESPACES + end + + def fix_namespace_route_names(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name AS correct_name + FROM + routes + INNER JOIN namespaces ON routes.source_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + namespaces.type IS NULL + AND routes.source_type = 'Namespace' + AND namespaces.id BETWEEN #{from_id} AND #{to_id} + AND (routes.name != users.name OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb new file mode 100644 index 00000000000..b84ff32e712 --- /dev/null +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes the routes.name for all user-projects that have names + # that don't start with the users name. + # For more info see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23272 + class FixUserProjectRouteNames + def perform(from_id, to_id) + ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE + WITH routes_to_update AS ( + SELECT + routes.id, + users.name || ' / ' || projects.name AS correct_name + FROM + routes + INNER JOIN projects ON routes.source_id = projects.id + INNER JOIN namespaces ON projects.namespace_id = namespaces.id + INNER JOIN users ON namespaces.owner_id = users.id + WHERE + routes.source_type = 'Project' + AND routes.id BETWEEN #{from_id} AND #{to_id} + AND namespaces.type IS NULL + AND (routes.name NOT LIKE users.name || '%' OR routes.name IS NULL) + ) + UPDATE + routes + SET + name = routes_to_update.correct_name + FROM + routes_to_update + WHERE + routes_to_update.id = routes.id + ROUTES_UPDATE + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index a4c6540c61b..eb4bc0aaf28 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -18,6 +18,14 @@ module Gitlab execute("INSERT INTO merge_request_assignees (merge_request_id, user_id) #{select_sql}") end + def perform_all_sync(batch_size:) + MergeRequest.each_batch(of: batch_size) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + perform(*range) + end + end + private def merge_request_assignees_not_exists_clause diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 755b5ee725a..d2924d10225 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -42,7 +42,7 @@ module Gitlab #{e.message} #{e.backtrace.join("\n ")} MSG - Rails.logger.error(msg) + Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 1ee44a3a5a9..cce2a82c098 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -111,7 +111,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb new file mode 100644 index 00000000000..61011abddf5 --- /dev/null +++ b/lib/gitlab/batch_pop_queueing.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + ## + # This class is a queuing system for processing expensive tasks in an atomic manner + # with batch poping to let you optimize the total processing time. + # + # In usual queuing system, the first item started being processed immediately + # and the following items wait until the next items have been popped from the queue. + # On the other hand, this queueing system, the former part is same, however, + # it pops the enqueued items as batch. This is especially useful when you want to + # drop redandant items from the queue in order to process important items only, + # thus it's more efficient than the traditional queueing system. + # + # Caveats: + # - The order of the items are not guaranteed because of `sadd` (Redis Sets). + # + # Example: + # ``` + # class TheWorker + # def perform + # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| + # item = extract_the_most_important_item_from(items_in_queue) + # expensive_process(item) + # end + # + # if result[:status] == :finished && result[:new_items].present? + # item = extract_the_most_important_item_from(items_in_queue) + # TheWorker.perform_async(item.id) + # end + # end + # end + # ``` + # + class BatchPopQueueing + attr_reader :namespace, :queue_id + + EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour + MAX_COUNTS_OF_POP_ALL = 1000 + + # Initialize queue + # + # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. + # @param [String] queue_id The identifier of the queue. + # @return [Boolean] + def initialize(namespace, queue_id) + raise ArgumentError if namespace.empty? || queue_id.empty? + + @namespace, @queue_id = namespace, queue_id + end + + ## + # Execute the given block in an exclusive lock. + # If there is the other thread has already working on the block, + # it enqueues the items without processing the block. + # + # @param [Array<String>] new_items New items to be added to the queue. + # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. + # @return [Hash] + # - status => One of the `:enqueued` or `:finished`. + # - new_items => Newly enqueued items during the given block had been processed. + # + # NOTE: If an exception is raised in the block, the poppped items will not be recovered. + # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. + def safe_execute(new_items, lock_timeout: 10.minutes, &block) + enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) + + lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) + + return { status: :enqueued } unless uuid = lease.try_obtain + + begin + all_args = pop_all + + yield all_args if block_given? + + { status: :finished, new_items: peek_all } + ensure + Gitlab::ExclusiveLease.cancel(lock_key, uuid) + end + end + + private + + def lock_key + @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" + end + + def queue_key + @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" + end + + def enqueue(items, expire_time) + Gitlab::Redis::Queues.with do |redis| + redis.sadd(queue_key, items) + redis.expire(queue_key, expire_time.to_i) + end + end + + def pop_all + Gitlab::Redis::Queues.with do |redis| + redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) + end + end + + def peek_all + Gitlab::Redis::Queues.with do |redis| + redis.smembers(queue_key) + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 8a84744aa2d..005ea4b7a46 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -22,6 +22,7 @@ code_quality: reports: codequality: gl-code-quality-report.json expire_in: 1 week + dependencies: [] only: - branches - tags diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index ee7164b3e55..808814c39e0 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -17,7 +17,7 @@ module Gitlab @limit = limit @dry_run = dry_run @niceness = niceness || DEFAULT_NICENESS - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger @total_found = @total_cleaned = 0 new_batch! diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb index 5c30258c0fc..53e0c83046e 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -22,7 +22,7 @@ module Gitlab attr_reader :batch_size, :dry_run attr_accessor :artifact_files - def initialize(batch_size:, dry_run: true, logger: Rails.logger) + def initialize(batch_size:, dry_run: true, logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger @batch_size = batch_size @dry_run = dry_run @logger = logger diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb index 2ee8b60e76a..5aace564c2d 100644 --- a/lib/gitlab/cleanup/project_upload_file_finder.rb +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -49,7 +49,7 @@ module Gitlab cmd = %W[#{ionice} -c Idle] + cmd if ionice log_msg = "find command: \"#{cmd.join(' ')}\"" - Rails.logger.info log_msg + Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger cmd end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 82a405362c2..056e075cb21 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :logger def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: true) diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 03298d960a4..42c93b7aecb 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -7,7 +7,7 @@ module Gitlab BATCH_SIZE = 100 def initialize(logger: nil) - @logger = logger || Rails.logger + @logger = logger || Rails.logger # rubocop:disable Gitlab/RailsLogger end def run!(dry_run: false) diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 469a7fd9f7b..32d5e4b9ea3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab # need to be added to the application settings. To prevent Rake tasks # and other callers from failing, use any loaded settings and return # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? + if ActiveRecord::Base.connection.migration_context.needs_migration? db_attributes = current_settings&.attributes || {} fake_application_settings(db_attributes) elsif current_settings.present? diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 34c1e6ad8ca..3e4c720b49a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -128,7 +128,7 @@ module Gitlab order = "#{field} IS NULL, #{order}" if direction == 'ASC' end - order + Arel.sql(order) end def self.nulls_first_order(field, direction = 'ASC') @@ -142,7 +142,7 @@ module Gitlab order = "#{field} IS NULL, #{order}" if direction == 'DESC' end - order + Arel.sql(order) end def self.random @@ -284,17 +284,28 @@ module Gitlab end # inside_transaction? will return true if the caller is running within a transaction. Handles special cases - # when running inside a test environment, in which the entire test is running with a DatabaseCleaner transaction + # when running inside a test environment, where tests may be wrapped in transactions def self.inside_transaction? - ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + if Rails.env.test? + ActiveRecord::Base.connection.open_transactions > open_transactions_baseline + else + ActiveRecord::Base.connection.open_transactions > 0 + end end - def self.open_transactions_baseline - if ::Rails.env.test? - return DatabaseCleaner.connections.count { |conn| conn.strategy.is_a?(DatabaseCleaner::ActiveRecord::Transaction) } - end + # These methods that access @open_transactions_baseline are not thread-safe. + # These are fine though because we only call these in RSpec's main thread. If we decide to run + # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value + def self.set_open_transactions_baseline + @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions + end + + def self.reset_open_transactions_baseline + @open_transactions_baseline = 0 + end - 0 + def self.open_transactions_baseline + @open_transactions_baseline ||= 0 end private_class_method :open_transactions_baseline @@ -310,7 +321,7 @@ module Gitlab gitlab_database_transaction_seconds.observe(labels, duration_seconds) rescue Prometheus::Client::LabelSetValidator::LabelSetError => err # Ensure that errors in recording these metrics don't affect the operation of the application - Rails.logger.error("Unable to observe database transaction duration: #{err}") + Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger end # MonkeyPatch for ActiveRecord::Base for adding observability diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 862ab96c887..26adf4e221b 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -24,7 +24,7 @@ module Gitlab begin from(nil) - .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')") + .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) .first rescue ActiveRecord::StatementInvalid # This error is raised when using a non-existing table name. In this diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e2cbf91f281..0c5f33e1b2a 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -61,7 +61,7 @@ module Gitlab end if index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -91,7 +91,7 @@ module Gitlab end unless index_exists?(table_name, column_name, options) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -121,7 +121,7 @@ module Gitlab end unless index_exists_by_name?(table_name, index_name) - Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" + Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}" # rubocop:disable Gitlab/RailsLogger return end @@ -149,6 +149,8 @@ module Gitlab # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # + # rubocop:disable Gitlab/RailsLogger def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. @@ -208,6 +210,7 @@ module Gitlab execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") end end + # rubocop:enable Gitlab/RailsLogger def foreign_key_exists?(source, target = nil, column: nil) foreign_keys(source).any? do |key| diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 6bbad707f0f..3e8a9b89998 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -70,7 +70,7 @@ module Gitlab unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message + Rails.logger.error message # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 580be9fe267..4dc7a62797a 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -56,7 +56,7 @@ module Gitlab unless gitlab_shell.mv_repository(project.repository_storage, old_path, new_path) - Rails.logger.error "Error moving #{old_path} to #{new_path}" + Rails.logger.error "Error moving #{old_path} to #{new_path}" # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb index 6b6b1d85109..58dc1527c7a 100644 --- a/lib/gitlab/email/hook/disable_email_interceptor.rb +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -7,7 +7,7 @@ module Gitlab def self.delivering_email(message) message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" + Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 5a61a7f5d60..88729babb2b 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -59,7 +59,7 @@ module Gitlab begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") # rubocop:disable Gitlab/RailsLogger '' end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 060a29be782..b7b7578cef9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -936,7 +936,7 @@ module Gitlab gitaly_repository_client.cleanup if exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") + Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") # rubocop:disable Gitlab/RailsLogger Gitlab::Metrics.counter( :failed_repository_cleanup_total, 'Number of failed repository cleanup events' diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cc9503fb6de..091351e5cb2 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -67,7 +67,7 @@ module Gitlab File.read(cert_file).scan(PEM_REGEX).map do |cert| OpenSSL::X509::Certificate.new(cert).to_pem rescue OpenSSL::OpenSSLError => e - Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Rails.logger.error "Could not load certificate #{cert_file} #{e}" # rubocop:disable Gitlab/RailsLogger Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) nil end.compact diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 6046e30d4ef..30763492235 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -29,7 +29,7 @@ module Gitlab yield object end rescue StandardError => e - Rails.logger.error("The Lfs import process failed. #{e.message}") + Rails.logger.error("The Lfs import process failed. #{e.message}") # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index a52866c4b08..929fceaacf2 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -40,7 +40,7 @@ module Gitlab pname = project.path_with_namespace - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger .info("GitHub importer finished updating repository for #{pname}") repository_updates_counter.increment diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb new file mode 100644 index 00000000000..ac2a78c0f28 --- /dev/null +++ b/lib/gitlab/graphql/docs/helper.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Helper with functions to be used by HAML templates + # This includes graphql-docs gem helpers class. + # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb + module Helper + include GraphQLDocs::Helpers + + def auto_generated_comment + <<-MD.strip_heredoc + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. + ---> + MD + end + + # Some fields types are arrays of other types and are displayed + # on docs wrapped in square brackets, for example: [String!]. + # This makes GitLab docs renderer thinks they are links so here + # we change them to be rendered as: String! => Array. + def render_field_type(type) + array_type = type[/\[(.+)\]/, 1] + + if array_type + "#{array_type} => Array" + else + type + end + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + graphql_object_types.select do |object_type| + !object_type[:name]["Connection"] && + !object_type[:name]["Edge"] && + !object_type[:name]["__"] + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb new file mode 100644 index 00000000000..f47a372aa19 --- /dev/null +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Gitlab renderer for graphql-docs. + # Uses HAML templates to parse markdown and generate .md files. + # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. + # + # Arguments: + # schema - the GraphQL schema defition. For GitLab should be: GitlabSchema.graphql_definition + # output_dir: The folder where the markdown files will be saved + # template: The path of the haml template to be parsed + class Renderer + include Gitlab::Graphql::Docs::Helper + + def initialize(schema, output_dir:, template:) + @output_dir = output_dir + @template = template + @layout = Haml::Engine.new(File.read(template)) + @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + end + + def render + contents = @layout.render(self) + + write_file(contents) + end + + private + + def write_file(contents) + filename = File.join(@output_dir, 'index.md') + + FileUtils.mkdir_p(@output_dir) + File.write(filename, contents) + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml new file mode 100644 index 00000000000..cc22d43ab4f --- /dev/null +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -0,0 +1,25 @@ +-# haml-lint:disable UnnecessaryStringOutput + += auto_generated_comment + +:plain + # GraphQL API Resources + + This documentation is self-generated based on GitLab current GraphQL schema. + + The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + + ## Objects +\ +- objects.each do |type| + - unless type[:fields].empty? + = "### #{type[:name]}" + \ + ~ "| Name | Type | Description |" + ~ "| --- | ---- | ---------- |" + - type[:fields].each do |field| + = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + \ + + + diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 1f0deebea39..6a8e16f5a85 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -62,6 +62,7 @@ module Gitlab # Flag a project to be migrated to Hashed Storage # # @param [Project] project that will be migrated + # rubocop:disable Gitlab/RailsLogger def migrate(project) Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." @@ -69,10 +70,12 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Flag a project to be rolled-back to Legacy Storage # # @param [Project] project that will be rolled-back + # rubocop:disable Gitlab/RailsLogger def rollback(project) Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." @@ -80,6 +83,7 @@ module Gitlab rescue => err Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end + # rubocop:enable Gitlab/RailsLogger # Returns whether we have any pending storage migration # diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 87a31a37e3f..14727b03ce9 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -19,8 +19,12 @@ module Gitlab ENV['ID_TO'] end + def self.using_ranges? + !range_from.nil? && !range_to.nil? + end + def self.range_single_item? - !range_from.nil? && range_from == range_to + using_ranges? && range_from == range_to end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 3588260d6eb..5a1e8c2a1dd 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -18,7 +18,7 @@ module Gitlab def metrics result, elapsed = with_timing(&method(:check)) - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless successful?(result) # rubocop:disable Gitlab/RailsLogger [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", successful?(result) ? 1 : 0), diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index deb2f59f05f..0b534a5bafc 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -43,7 +43,7 @@ module Gitlab target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref) - Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") + Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") # rubocop:disable Gitlab/RailsLogger end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 20caadb89c0..dec99c23a2d 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -20,7 +20,7 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 72f575db095..bea7a7cce65 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") + Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 6d978d00ea5..86ea7a30e69 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -36,7 +36,7 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') end end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index de14df56555..1350924cd76 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -59,6 +59,13 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client + delegate :create_role, + :get_role, + :update_role, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client delegate :create_role_binding, :get_role_binding, :update_role_binding, diff --git a/lib/gitlab/kubernetes/role.rb b/lib/gitlab/kubernetes/role.rb new file mode 100644 index 00000000000..096f60f0372 --- /dev/null +++ b/lib/gitlab/kubernetes/role.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Role + def initialize(name:, namespace:, rules:) + @name = name + @namespace = namespace + @rules = rules + end + + def generate + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + private + + attr_reader :name, :namespace, :rules + end + end +end diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index cb0cb42d007..0404fb4453c 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,10 @@ module Gitlab module Kubernetes class RoleBinding - def initialize(name:, role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, role_kind:, namespace:, service_account_name:) @name = name @role_name = role_name + @role_kind = role_kind @namespace = namespace @service_account_name = service_account_name end @@ -20,7 +21,7 @@ module Gitlab private - attr_reader :name, :role_name, :namespace, :service_account_name + attr_reader :name, :role_name, :role_kind, :namespace, :service_account_name def metadata { name: name, namespace: namespace } @@ -29,7 +30,7 @@ module Gitlab def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb index f3abe631779..233d3bf1ac7 100644 --- a/lib/gitlab/markdown_cache/active_record/extension.rb +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -26,10 +26,6 @@ module Gitlab attrs end - def changed_markdown_fields - changed_attributes.keys.map(&:to_s) & cached_markdown_fields.markdown_fields.map(&:to_s) - end - def write_markdown_field(field_name, value) write_attribute(field_name, value) end diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index 97fc23343b4..af3237f4ba6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -36,8 +36,8 @@ module Gitlab false end - def changed_markdown_fields - [] + def changed_attributes + {} end def cached_markdown diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 6a062e93f0f..d7d848d2833 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -19,7 +19,7 @@ module Gitlab def safe_sample sample rescue => e - Rails.logger.warn("#{self.class}: #{e}, stopping") + Rails.logger.warn("#{self.class}: #{e}, stopping") # rubocop:disable Gitlab/RailsLogger stop end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 25e40c70230..4e835f37c04 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -43,7 +43,7 @@ module Gitlab def puma_stats Puma.stats rescue NoMethodError - Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" + Rails.logger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" # rubocop:disable Gitlab/RailsLogger nil end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 5c2f07b95e2..51f48095cb5 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -63,10 +63,6 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end - - def self.clk_tck - @clk_tck ||= `getconf CLK_TCK`.to_i - end end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 817db12ac55..53e55269c6e 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -25,7 +25,7 @@ module Gitlab def call if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') # rubocop:disable Gitlab/RailsLogger if json_request? return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb index d74fdba2241..281eafb142f 100644 --- a/lib/gitlab/pages_client.rb +++ b/lib/gitlab/pages_client.rb @@ -110,7 +110,7 @@ module Gitlab end rescue Errno::EACCES => ex # TODO stop rescuing this exception in GitLab 11.0 https://gitlab.com/gitlab-org/gitlab-ce/issues/45672 - Rails.logger.error("Could not write pages admin token file: #{ex}") + Rails.logger.error("Could not write pages admin token file: #{ex}") # rubocop:disable Gitlab/RailsLogger rescue Errno::EEXIST # Another process wrote the token file concurrently with us. Use their token, not ours. end diff --git a/lib/gitlab/patch/active_record_query_cache.rb b/lib/gitlab/patch/active_record_query_cache.rb new file mode 100644 index 00000000000..71d66bdbe02 --- /dev/null +++ b/lib/gitlab/patch/active_record_query_cache.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Fixes a bug where the query cache isn't aware of the shared +# ActiveRecord connection used in tests +# https://github.com/rails/rails/issues/36587 + +# To be removed with https://gitlab.com/gitlab-org/gitlab-ce/issues/64413 + +module Gitlab + module Patch + module ActiveRecordQueryCache + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def enable_query_cache! + @query_cache_enabled[connection_cache_key(current_thread)] = true + connection.enable_query_cache! if active_connection? + end + + def disable_query_cache! + @query_cache_enabled.delete connection_cache_key(current_thread) + connection.disable_query_cache! if active_connection? + end + + def query_cache_enabled + @query_cache_enabled[connection_cache_key(current_thread)] + end + + def active_connection? + @thread_cached_conns[connection_cache_key(current_thread)] + end + + private + + def current_thread + @lock_thread || Thread.current + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end +end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb index fa8b37b20ca..6a2841b6a8e 100644 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -9,9 +9,15 @@ module Gitlab def get_gitlab_model(phabricator_id) cached_info = get(phabricator_id) - return unless cached_info[:classname] && cached_info[:database_id] - cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + if cached_info[:classname] && cached_info[:database_id] + object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + else + object = yield if block_given? + set_gitlab_model(object, phabricator_id) if object + end + + object end def set_gitlab_model(object, phabricator_id) diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb new file mode 100644 index 00000000000..fc8c3f7cde9 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/user.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class User + MAX_PAGE_SIZE = 100 + + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def users(phids) + phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) } + end + + private + + def get_page(phids) + UsersResponse.new(get_users(phids)) + end + + def get_users(phids) + client.get('user.search', + params: { constraints: { phids: phids } }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb new file mode 100644 index 00000000000..3dfb29a7be5 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/users_response.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Conduit + class UsersResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + def users + @users ||= conduit_response.data.map do |user_json| + Gitlab::PhabricatorImport::Representation::User.new(user_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index 40d4392cbc1..77ee11c7cdd 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -8,9 +8,7 @@ module Gitlab end def execute - # TODO: get the user from the project namespace from the username loaded by Phab-id - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 - issue.author = User.ghost + issue.author = user_finder.find(task.author_phid) || User.ghost # TODO: Reformat the description with attachments, escaping accidental # links and add attachments @@ -19,6 +17,10 @@ module Gitlab save! + if owner = user_finder.find(task.owner_phid) + issue.assignees << owner + end + issue end @@ -41,6 +43,10 @@ module Gitlab project.issues.new end + def user_finder + @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids) + end + def find_issue_by_phabricator_id(phabricator_id) object_map.get_gitlab_model(phabricator_id) end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb index 6aedc71b626..ba93fb37a8e 100644 --- a/lib/gitlab/phabricator_import/representation/task.rb +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -11,6 +11,18 @@ module Gitlab json['phid'] end + def author_phid + json['fields']['authorPHID'] + end + + def owner_phid + json['fields']['ownerPHID'] + end + + def phids + @phids ||= [author_phid, owner_phid] + end + def issue_attributes @issue_attributes ||= { title: issue_title, diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb new file mode 100644 index 00000000000..7fd7cecc6ae --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/user.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Representation + class User + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def username + json['fields']['username'] + end + + private + + attr_reader :json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb new file mode 100644 index 00000000000..4b50431e0e0 --- /dev/null +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class UserFinder + def initialize(project, phids) + @project, @phids = project, phids + @loaded_phids = Set.new + end + + def find(phid) + found_user = object_map.get_gitlab_model(phid) do + find_user_for_phid(phid) + end + + loaded_phids << phid + + found_user + end + + private + + attr_reader :project, :phids, :loaded_phids + + def object_map + @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project) + end + + def find_user_for_phid(phid) + phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid } + return unless phabricator_user + + project.authorized_users.find_by_username(phabricator_user.username) + end + + def phabricator_users + @user_responses ||= client.users(users_to_request).flat_map(&:users) + end + + def users_to_request + phids - loaded_phids.to_a + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::User + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end +end diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index d2dbc6f5ef5..1c43de35816 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -22,6 +22,7 @@ module Gitlab end end + # rubocop:disable Gitlab/RailsLogger def decrease redis_cmd do |redis| current_value = redis.decr(key) @@ -32,6 +33,7 @@ module Gitlab end end end + # rubocop:enable Gitlab/RailsLogger private @@ -39,7 +41,7 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| yield(redis) } true rescue => e - Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") + Rails.logger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") # rubocop:disable Gitlab/RailsLogger false end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 7a1a2eaf6c0..e43147a3f37 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -4,14 +4,6 @@ module Gitlab module Regex extend self - def namespace_name_regex - @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze - end - - def namespace_name_regex_message - "can contain only letters, digits, '_', '.', dash and space." - end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 931298b5117..e40c366ed02 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -145,7 +145,7 @@ module Gitlab def expire_method_caches(methods) methods.each do |name| unless cached_methods.include?(name.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" # rubocop:disable Gitlab/RailsLogger next end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index 0928ccdc324..bb4e4ce7bbc 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -48,7 +48,7 @@ module Gitlab attr_reader :logger - def initialize(logger: Rails.logger) + def initialize(logger: Rails.logger) # rubocop:disable Gitlab/RailsLogger @logger = logger end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 93182607616..0fa17b3f559 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -78,7 +78,7 @@ module Gitlab true rescue => err # Once the Rugged codes gets removes this can be improved - Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") + Rails.logger.error("Failed to add repository #{storage}/#{disk_path}: #{err}") # rubocop:disable Gitlab/RailsLogger false end @@ -153,7 +153,7 @@ module Gitlab !!rm_directory(storage, "#{name}.git") rescue ArgumentError => e - Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") # rubocop:disable Gitlab/RailsLogger false end @@ -238,7 +238,7 @@ module Gitlab def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? - Rails.logger.info("Removing keys not found in DB") + Rails.logger.info("Removing keys not found in DB") # rubocop:disable Gitlab/RailsLogger batch_read_key_ids do |ids_in_file| ids_in_file.uniq! @@ -248,7 +248,7 @@ module Gitlab ids_to_remove = ids_in_file - keys_in_db.pluck(:id) ids_to_remove.each do |id| - Rails.logger.info("Removing key-#{id} not found in DB") + Rails.logger.info("Removing key-#{id} not found in DB") # rubocop:disable Gitlab/RailsLogger remove_key("key-#{id}") end end @@ -368,7 +368,7 @@ module Gitlab return true if status.zero? - Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") # rubocop:disable Gitlab/RailsLogger false end @@ -465,7 +465,7 @@ module Gitlab end def logger - Rails.logger + Rails.logger # rubocop:disable Gitlab/RailsLogger end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9a8df719827..f6b2e2acf16 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -18,7 +18,6 @@ module Gitlab # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. # # Returns an array with [<uri>, <original-hostname>]. - # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/ParameterLists def validate!( url, @@ -30,7 +29,6 @@ module Gitlab enforce_user: false, enforce_sanitization: false, dns_rebind_protection: true) - # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Metrics/ParameterLists return [nil, nil] if url.nil? @@ -38,36 +36,31 @@ module Gitlab # Param url can be a string, URI or Addressable::URI uri = parse_url(url) - validate_html_tags!(uri) if enforce_sanitization + validate_uri( + uri: uri, + schemes: schemes, + ports: ports, + enforce_sanitization: enforce_sanitization, + enforce_user: enforce_user, + ascii_only: ascii_only + ) hostname = uri.hostname port = get_port(uri) - unless internal?(uri) - validate_scheme!(uri.scheme, schemes) - validate_port!(port, ports) if ports.any? - validate_user!(uri.user) if enforce_user - validate_hostname!(hostname) - validate_unicode_restriction!(uri) if ascii_only - end - - begin - addrs_info = Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| - addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr - end - rescue SocketError - return [uri, nil] - end + address_info = get_address_info(hostname, port) + return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) + protected_uri_with_hostname = enforce_uri_hostname(address_info, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) - validate_localhost!(addrs_info) unless allow_localhost - validate_loopback!(addrs_info) unless allow_localhost - validate_local_network!(addrs_info) unless allow_local_network - validate_link_local!(addrs_info) unless allow_local_network + validate_local_request( + address_info: address_info, + allow_localhost: allow_localhost, + allow_local_network: allow_local_network + ) protected_uri_with_hostname end @@ -101,11 +94,44 @@ module Gitlab [uri, hostname] end + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) + validate_html_tags(uri) if enforce_sanitization + + return if internal?(uri) + + validate_scheme(uri.scheme, schemes) + validate_port(get_port(uri), ports) if ports.any? + validate_user(uri.user) if enforce_user + validate_hostname(uri.hostname) + validate_unicode_restriction(uri) if ascii_only + end + + def get_address_info(hostname, port) + Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end + rescue SocketError + end + + def validate_local_request(address_info:, allow_localhost:, allow_local_network:) + return if allow_local_network && allow_localhost + + unless allow_localhost + validate_localhost(address_info) + validate_loopback(address_info) + end + + unless allow_local_network + validate_local_network(address_info) + validate_link_local(address_info) + end + end + def get_port(uri) uri.port || uri.default_port end - def validate_html_tags!(uri) + def validate_html_tags(uri) uri_str = uri.to_s sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) if sanitized_uri != uri_str @@ -125,7 +151,7 @@ module Gitlab CGI.unescape(url.to_s) =~ /\n|\r/ end - def validate_port!(port, ports) + def validate_port(port, ports) return if port.blank? # Only ports under 1024 are restricted return if port >= 1024 @@ -134,20 +160,20 @@ module Gitlab raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" end - def validate_scheme!(scheme, schemes) + def validate_scheme(scheme, schemes) if scheme.blank? || (schemes.any? && !schemes.include?(scheme)) raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end - def validate_user!(value) + def validate_user(value) return if value.blank? return if value =~ /\A\p{Alnum}/ raise BlockedUrlError, "Username needs to start with an alphanumeric character" end - def validate_hostname!(value) + def validate_hostname(value) return if value.blank? return if IPAddress.valid?(value) return if value =~ /\A\p{Alnum}/ @@ -155,13 +181,13 @@ module Gitlab raise BlockedUrlError, "Hostname or IP address invalid" end - def validate_unicode_restriction!(uri) + def validate_unicode_restriction(uri) return if uri.to_s.ascii_only? raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" end - def validate_localhost!(addrs_info) + def validate_localhost(addrs_info) local_ips = ["::", "0.0.0.0"] local_ips.concat(Socket.ip_address_list.map(&:ip_address)) @@ -170,19 +196,19 @@ module Gitlab raise BlockedUrlError, "Requests to localhost are not allowed" end - def validate_loopback!(addrs_info) + def validate_loopback(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } raise BlockedUrlError, "Requests to loopback addresses are not allowed" end - def validate_local_network!(addrs_info) + def validate_local_network(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } raise BlockedUrlError, "Requests to the local network are not allowed" end - def validate_link_local!(addrs_info) + def validate_link_local(addrs_info) netmask = IPAddr.new('169.254.0.0/16') return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 722e3e04d1c..eea7daa3d8e 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -41,7 +41,7 @@ module Mattermost begin yield self rescue Errno::ECONNREFUSED => e - Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) + Rails.logger.error(e.message + "\n" + e.backtrace.join("\n")) # rubocop:disable Gitlab/RailsLogger raise Mattermost::NoSessionError ensure destroy diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index c7dec09ba6b..a7dcd322e27 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -20,7 +20,7 @@ module MicrosoftTeams result = true if response rescue Gitlab::HTTP::Error, StandardError => error - Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") # rubocop:disable Gitlab/RailsLogger end result diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 19cc0baa2d3..bf15130d17e 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -32,6 +32,7 @@ module RspecFlaky flaky_examples[current_example.uid] = flaky_example end + # rubocop:disable Gitlab/RailsLogger def dump_summary(_) RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) @@ -45,6 +46,7 @@ module RspecFlaky # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end + # rubocop:enable Gitlab/RailsLogger private diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 105ef417df3..88172e26c67 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -147,6 +147,7 @@ namespace :gitlab do ENV['NICENESS'].presence end + # rubocop:disable Gitlab/RailsLogger def logger return @logger if defined?(@logger) @@ -159,5 +160,6 @@ namespace :gitlab do Rails.logger end end + # rubocop:enable Gitlab/RailsLogger end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake new file mode 100644 index 00000000000..c53d55ceea2 --- /dev/null +++ b/lib/tasks/gitlab/graphql.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference").freeze + TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/'.freeze + + namespace :graphql do + desc 'GitLab | Generate GraphQL docs' + task compile_docs: :environment do + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + + renderer.render + + puts "Documentation compiled." + end + end +end + +def render_options + { + output_dir: OUTPUT_DIR, + template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml') + } +end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 954f827f716..ccc96b7edfb 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -3,50 +3,44 @@ namespace :gitlab do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.rollback_pending? - warn "There is already a rollback operation in progress, " \ + abort "There is already a rollback operation in progress, " \ "running a migration at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - warn "There are no projects requiring storage migration with ID=#{helper.range_from}" - - next + abort "There are no projects requiring storage migration with ID=#{helper.range_from}" end puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." storage_migrator.migrate(project) + else + legacy_projects_count = if helper.using_ranges? + Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count + else + Project.with_unmigrated_storage.count + end + + if legacy_projects_count == 0 + abort 'There are no projects requiring storage migration. Nothing to do!' + end - next - end - - legacy_projects_count = Project.with_unmigrated_storage.count - - if legacy_projects_count == 0 - warn 'There are no projects requiring storage migration. Nothing to do!' - - next - end - - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' @@ -55,50 +49,44 @@ namespace :gitlab do desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' task rollback_to_legacy: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.migration_pending? - warn "There is already a migration operation in progress, " \ + abort "There is already a migration operation in progress, " \ "running a rollback at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) unless project - warn "There are no projects that can be rolledback with ID=#{helper.range_from}" - - next + abort "There are no projects that can be rolledback with ID=#{helper.range_from}" end puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." storage_migrator.rollback(project) + else + hashed_projects_count = if helper.using_ranges? + Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count + else + Project.with_storage_feature(:repository).count + end + + if hashed_projects_count == 0 + abort 'There are no projects that can have storage rolledback. Nothing to do!' + end - next - end - - hashed_projects_count = Project.with_storage_feature(:repository).count - - if hashed_projects_count == 0 - warn 'There are no projects that can have storage rolledback. Nothing to do!' - - next - end - - print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_rollback do |start, finish| - storage_migrator.bulk_schedule_rollback(start: start, finish: finish) + helper.project_id_batches_rollback do |start, finish| + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' diff --git a/package.json b/package.json index 5955790488a..44aa850860e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@babel/preset-env": "^7.4.4", "@gitlab/csslab": "^1.9.0", "@gitlab/svgs": "^1.67.0", - "@gitlab/ui": "^5.5.0", + "@gitlab/ui": "^5.6.0", "apollo-cache-inmemory": "^1.5.1", "apollo-client": "^2.5.1", "apollo-link": "^1.2.11", diff --git a/qa/README.md b/qa/README.md index 124a79a36b4..bab19665dac 100644 --- a/qa/README.md +++ b/qa/README.md @@ -75,14 +75,14 @@ You can also supply specific tests to run as another parameter. For example, to run the repository-related specs, you can execute: ``` -bundle exec bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/repository +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- qa/specs/features/browser_ui/3_create/repository ``` Since the arguments would be passed to `rspec`, you could use all `rspec` options there. For example, passing `--backtrace` and also line number: ``` -bundle exec bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb:6 --backtrace +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb:6 --backtrace ``` Note that the separator `--` is required; all subsequent options will be @@ -140,7 +140,7 @@ tests that are expected to fail while a fix is in progress (similar to how can be used). ``` -bundle exec bin/qa Test::Instance::All http://localhost -- --tag quarantine +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- --tag quarantine ``` If `quarantine` is used with other tags, tests will only be run if they have at @@ -159,7 +159,7 @@ option `--enable-feature FEATURE_FLAG`. For example, to enable the feature flag that enforces Gitaly request limits, you would use the command: ``` -bundle exec bin/qa Test::Instance::All http://localhost --enable-feature gitaly_enforce_requests_limits +bundle exec bin/qa Test::Instance::All http://localhost:3000 --enable-feature gitaly_enforce_requests_limits ``` This will instruct the QA framework to enable the `gitaly_enforce_requests_limits` diff --git a/qa/qa/page/base.rb b/qa/qa/page/base.rb index d0fe2987b0a..130e5e33ab4 100644 --- a/qa/qa/page/base.rb +++ b/qa/qa/page/base.rb @@ -25,19 +25,10 @@ module QA end end - def retry_until(max_attempts: 3, reload: false) - attempts = 0 - - while attempts < max_attempts - result = yield - return result if result - - refresh if reload - - attempts += 1 + def retry_until(max_attempts: 3, reload: false, sleep_interval: 0) + QA::Support::Retrier.retry_until(max_attempts: max_attempts, reload: reload, sleep_interval: sleep_interval) do + yield end - - false end def retry_on_exception(max_attempts: 3, reload: false, sleep_interval: 0.5) diff --git a/qa/qa/page/component/note.rb b/qa/qa/page/component/note.rb index 07e191f1c9b..fe324574f4d 100644 --- a/qa/qa/page/component/note.rb +++ b/qa/qa/page/component/note.rb @@ -10,6 +10,10 @@ module QA element :discussion_option end + base.view 'app/assets/javascripts/notes/components/note_actions.vue' do + element :note_edit_button + end + base.view 'app/assets/javascripts/notes/components/note_form.vue' do element :reply_input element :reply_comment_button @@ -49,6 +53,12 @@ module QA def expand_replies click_element :expand_replies end + + def edit_comment(text) + click_element :note_edit_button + fill_element :reply_input, text + click_element :reply_comment_button + end end end end diff --git a/qa/qa/page/element.rb b/qa/qa/page/element.rb index 7a01320901d..9e6fd2fdd4f 100644 --- a/qa/qa/page/element.rb +++ b/qa/qa/page/element.rb @@ -28,7 +28,7 @@ module QA end def selector_css - ".#{selector}" + %Q([data-qa-selector="#{@name}"],.#{selector}) end def expression @@ -40,7 +40,7 @@ module QA end def matches?(line) - !!(line =~ expression) + !!(line =~ /["']#{name}['"]|#{expression}/) end end end diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index 5eb24d2d2ba..c98d85d7911 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -5,14 +5,14 @@ module QA module Main class Menu < Page::Base view 'app/views/layouts/header/_current_user_dropdown.html.haml' do - element :user_sign_out_link, 'link_to _("Sign out")' # rubocop:disable QA/ElementWithPattern + element :sign_out_link element :settings_link, 'link_to s_("CurrentUser|Settings")' # rubocop:disable QA/ElementWithPattern end view 'app/views/layouts/header/_default.html.haml' do element :navbar, required: true element :user_avatar, required: true - element :user_menu, '.dropdown-menu' # rubocop:disable QA/ElementWithPattern + element :user_menu, required: true end view 'app/views/layouts/nav/_dashboard.html.haml' do @@ -53,7 +53,7 @@ module QA def sign_out within_user_menu do - click_link 'Sign out' + click_element :sign_out_link end end @@ -82,7 +82,7 @@ module QA private def within_top_menu - page.within('.qa-navbar') do + within_element(:navbar) do yield end end @@ -91,7 +91,7 @@ module QA within_top_menu do click_element :user_avatar - page.within('.dropdown-menu') do + within_element(:user_menu) do yield end end diff --git a/qa/qa/runtime/logger.rb b/qa/qa/runtime/logger.rb index bd5c4fe5bf5..7f73f1bd01b 100644 --- a/qa/qa/runtime/logger.rb +++ b/qa/qa/runtime/logger.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'logger' +require 'forwardable' module QA module Runtime diff --git a/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb b/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb index cf225a639b6..5cd6bac3f5a 100644 --- a/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb +++ b/qa/qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb @@ -11,8 +11,10 @@ module QA expect(menu).to have_personal_area end - Page::Main::Menu.perform do |menu| - menu.sign_out + Support::Retrier.retry_until(reload: false, sleep_interval: 0.5) do + Page::Main::Menu.perform(&:sign_out) + + Page::Main::Login.perform(&:has_sign_in_tab?) end Page::Main::Login.perform do |form| diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb new file mode 100644 index 00000000000..a62a51b11f4 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module QA + context 'Plan' do + describe 'Issue comments' do + it 'user comments on an issue and edits the comment' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + issue = Resource::Issue.fabricate_via_api! do |issue| + issue.title = 'issue title' + end + issue.visit! + + Page::Project::Issue::Show.perform do |issue_show_page| + first_version_of_comment = 'First version of the comment' + second_version_of_comment = 'Second version of the comment' + + issue_show_page.comment(first_version_of_comment) + + expect(issue_show_page).to have_content(first_version_of_comment) + + issue_show_page.edit_comment(second_version_of_comment) + + expect(issue_show_page).to have_content(second_version_of_comment) + expect(issue_show_page).not_to have_content(first_version_of_comment) + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb index 2b3d9b1711d..36cbd1b81f0 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb @@ -56,7 +56,7 @@ module QA project.visit! end - it 'branches are correctly listed after CRUD operations' do + it 'lists branches correctly after CRUD operations' do Page::Project::Menu.perform(&:go_to_repository_branches) expect(page).to have_content(master_branch) diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb index b060f15168c..b2c70547421 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb @@ -7,12 +7,12 @@ module QA Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) - project = Resource::Project.fabricate! do |project| + project = Resource::Project.fabricate_via_api! do |project| project.name = 'project-with-ci-variables' project.description = 'project with CI variables' end - Resource::CiVariable.fabricate! do |resource| + Resource::CiVariable.fabricate_via_api! do |resource| resource.project = project resource.key = 'VARIABLE_KEY' resource.value = 'some_CI_variable' diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb index caa9be341b4..791dc62e32f 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb @@ -10,7 +10,7 @@ module QA deploy_token_name = 'deploy token name' one_week_from_now = Date.today + 7 - deploy_token = Resource::DeployToken.fabricate! do |resource| + deploy_token = Resource::DeployToken.fabricate_via_browser_ui! do |resource| resource.name = deploy_token_name resource.expires_at = one_week_from_now end diff --git a/qa/qa/support/retrier.rb b/qa/qa/support/retrier.rb index 8be4e9f5365..230cec8f8d2 100644 --- a/qa/qa/support/retrier.rb +++ b/qa/qa/support/retrier.rb @@ -23,6 +23,25 @@ module QA raise end end + + def retry_until(max_attempts: 3, reload: false, sleep_interval: 0) + QA::Runtime::Logger.debug("with retry_until: max_attempts #{max_attempts}; sleep_interval #{sleep_interval}; reload:#{reload}") + attempts = 0 + + while attempts < max_attempts + QA::Runtime::Logger.debug("Attempt number #{attempts + 1}") + result = yield + return result if result + + sleep sleep_interval + + refresh if reload + + attempts += 1 + end + + false + end end end end diff --git a/qa/spec/page/element_spec.rb b/qa/spec/page/element_spec.rb index f746fe06e40..20d4a00c020 100644 --- a/qa/spec/page/element_spec.rb +++ b/qa/spec/page/element_spec.rb @@ -11,7 +11,7 @@ describe QA::Page::Element do describe '#selector_css' do it 'transforms element name into QA-specific clickable css selector' do expect(described_class.new(:sign_in_button).selector_css) - .to eq '.qa-sign-in-button' + .to include('.qa-sign-in-button') end end @@ -49,6 +49,10 @@ describe QA::Page::Element do it 'does not match if QA selector is not there' do expect(subject.matches?('some_name selector')).to be false end + + it 'matches when element name is specified' do + expect(subject.matches?('data:{qa:{selector:"some_name"}}')).to be true + end end describe 'attributes' do @@ -106,4 +110,11 @@ describe QA::Page::Element do end end end + + describe 'data-qa selectors' do + subject { described_class.new(:my_element) } + it 'properly translates to a data-qa-selector' do + expect(subject.selector_css).to include(%q([data-qa-selector="my_element"])) + end + end end diff --git a/rubocop/cop/gitlab/rails_logger.rb b/rubocop/cop/gitlab/rails_logger.rb new file mode 100644 index 00000000000..d1a06a9a100 --- /dev/null +++ b/rubocop/cop/gitlab/rails_logger.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require_relative '../../code_reuse_helpers' + +module RuboCop + module Cop + module Gitlab + class RailsLogger < ::RuboCop::Cop::Cop + include CodeReuseHelpers + + # This cop checks for the Rails.logger in the codebase + # + # @example + # + # # bad + # Rails.logger.error("Project #{project.full_path} could not be saved") + # + # # good + # Gitlab::AppLogger.error("Project %{project_path} could not be saved" % { project_path: project.full_path }) + MSG = 'Use a structured JSON logger instead of `Rails.logger`. ' \ + 'https://docs.gitlab.com/ee/development/logging.html'.freeze + + def_node_matcher :rails_logger?, <<~PATTERN + (send (const nil? :Rails) :logger ... ) + PATTERN + + WHITELISTED_DIRECTORIES = %w[ + spec + ].freeze + + def on_send(node) + return if in_whitelisted_directory?(node) + return unless rails_logger?(node) + + add_offense(node, location: :expression) + end + + def in_whitelisted_directory?(node) + path = file_path_for_node(node) + + WHITELISTED_DIRECTORIES.any? do |directory| + path.start_with?( + File.join(rails_root, directory), + File.join(rails_root, 'ee', directory) + ) + end + end + end + end + end +end diff --git a/rubocop/cop/qa/element_with_pattern.rb b/rubocop/cop/qa/element_with_pattern.rb index d14eeaaeaf3..d48f4725488 100644 --- a/rubocop/cop/qa/element_with_pattern.rb +++ b/rubocop/cop/qa/element_with_pattern.rb @@ -30,7 +30,7 @@ module RuboCop return if args.first.nil? args.first.each_node(:str) do |arg| - add_offense(arg, message: MESSAGE % "qa-#{element_name.value.to_s.tr('_', '-')}") + add_offense(arg, message: MESSAGE % "data-qa-selector=#{element_name.value}") end end diff --git a/rubocop/rubocop.rb b/rubocop/rubocop.rb index 27c63d92ae5..ba61a634d97 100644 --- a/rubocop/rubocop.rb +++ b/rubocop/rubocop.rb @@ -3,6 +3,7 @@ require_relative 'cop/gitlab/predicate_memoization' require_relative 'cop/gitlab/httparty' require_relative 'cop/gitlab/finder_with_find_by' require_relative 'cop/gitlab/union' +require_relative 'cop/gitlab/rails_logger' require_relative 'cop/include_action_view_context' require_relative 'cop/include_sidekiq_worker' require_relative 'cop/safe_params' diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data new file mode 100755 index 00000000000..9992a83e6a6 --- /dev/null +++ b/scripts/gather-test-memory-data @@ -0,0 +1,21 @@ +#!/usr/bin/env ruby + +require 'csv' + +def join_csv_files(output_path, input_paths) + return if input_paths.empty? + + input_csvs = input_paths.map do |input_path| + CSV.read(input_path, headers: true) + end + + CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv| + input_csvs.each do |input_csv| + input_csv.each do |line| + output_csv << line + end + end + end +end + +join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort) diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs index 0fc8f6fbd4b..83841512f1c 100755 --- a/scripts/trigger-build-docs +++ b/scripts/trigger-build-docs @@ -13,7 +13,7 @@ end # # The remote docs project # -GITLAB_DOCS_REPO = 'gitlab-com/gitlab-docs'.freeze +GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze # # Truncate the remote docs branch name otherwise we hit the filesystem @@ -31,7 +31,7 @@ end # to avoid race conditions, since a triggered pipeline will also run right # after the branch creation. This only happens the very first time a branch # is created and will be skipped in subsequent runs. Read more in -# https://gitlab.com/gitlab-com/gitlab-docs/issues/154. +# https://gitlab.com/gitlab-org/gitlab-docs/issues/154. # def create_remote_branch Gitlab.create_branch(GITLAB_DOCS_REPO, docs_branch, 'master') @@ -81,7 +81,7 @@ def slug end # -# Overriding vars in https://gitlab.com/gitlab-com/gitlab-docs/blob/master/.gitlab-ci.yml +# Overriding vars in https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml # def param_name "BRANCH_#{slug.upcase}" diff --git a/spec/controllers/admin/runners_controller_spec.rb b/spec/controllers/admin/runners_controller_spec.rb index 78c5e2a2656..bbeda7dae0f 100644 --- a/spec/controllers/admin/runners_controller_spec.rb +++ b/spec/controllers/admin/runners_controller_spec.rb @@ -23,10 +23,11 @@ describe Admin::RunnersController do control_count = ActiveRecord::QueryRecorder.new { get :index }.count - create(:ci_runner, :tagged_only) + create_list(:ci_runner, 5, :tagged_only) # There is still an N+1 query for `runner.builds.count` - expect { get :index }.not_to exceed_query_limit(control_count + 1) + # We also need to add 1 because it takes 2 queries to preload tags + expect { get :index }.not_to exceed_query_limit(control_count + 6) expect(response).to have_gitlab_http_status(200) expect(response.body).to have_content('tag1') diff --git a/spec/controllers/boards/issues_controller_spec.rb b/spec/controllers/boards/issues_controller_spec.rb index c84bb913cad..6cad060d888 100644 --- a/spec/controllers/boards/issues_controller_spec.rb +++ b/spec/controllers/boards/issues_controller_spec.rb @@ -164,6 +164,201 @@ describe Boards::IssuesController do end end + describe 'PUT move_multiple' do + let(:todo) { create(:group_label, group: group, name: 'Todo') } + let(:development) { create(:group_label, group: group, name: 'Development') } + let(:user) { create(:group_member, :maintainer, user: create(:user), group: group ).user } + let(:guest) { create(:group_member, :guest, user: create(:user), group: group ).user } + let(:project) { create(:project, group: group) } + let(:group) { create(:group) } + let(:board) { create(:board, project: project) } + let(:list1) { create(:list, board: board, label: todo, position: 0) } + let(:list2) { create(:list, board: board, label: development, position: 1) } + let(:issue1) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 10) } + let(:issue2) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 20) } + let(:issue3) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 30) } + let(:issue4) { create(:labeled_issue, project: project, labels: [development], author: user, relative_position: 100) } + + let(:move_params) do + { + board_id: board.id, + ids: [issue1.id, issue2.id, issue3.id], + from_list_id: list1.id, + to_list_id: list2.id, + move_before_id: issue4.id, + move_after_id: nil + } + end + + before do + project.add_maintainer(user) + project.add_guest(guest) + end + + shared_examples 'move issues endpoint provider' do + before do + sign_in(signed_in_user) + end + + it 'moves issues as expected' do + put :bulk_move, params: move_issues_params + expect(response).to have_gitlab_http_status(expected_status) + + list_issues user: requesting_user, board: board, list: list2 + expect(response).to have_gitlab_http_status(200) + + expect(response).to match_response_schema('entities/issue_boards') + + responded_issues = json_response['issues'] + expect(responded_issues.length).to eq expected_issue_count + + ids_in_order = responded_issues.pluck('id') + expect(ids_in_order).to eq(expected_issue_ids_in_order) + end + end + + context 'when items are moved to another list' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) { move_params } + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue4.id, issue1.id, issue2.id, issue3.id] } + end + end + + context 'when moving just one issue' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = [issue2.id] + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 2 } + let(:expected_issue_ids_in_order) { [issue4.id, issue2.id] } + end + end + + context 'when user is not allowed to move issue' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { guest } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = [issue2.id] + end + end + let(:requesting_user) { user } + let(:expected_status) { 403 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + + context 'when issues should be moved visually above existing issue in list' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_after_id] = issue4.id + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when destination list is empty' do + before do + # Remove issue from list + issue4.labels -= [development] + issue4.save! + end + + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 3 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id] } + end + end + + context 'when no position arguments are given' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when move_before_id and move_after_id are given' do + let(:issue5) { create(:labeled_issue, project: project, labels: [development], author: user, relative_position: 90) } + + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = issue5.id + hash[:move_after_id] = issue4.id + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 5 } + let(:expected_issue_ids_in_order) { [issue5.id, issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when request contains too many issues' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = (0..51).to_a + end + end + let(:requesting_user) { user } + let(:expected_status) { 422 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + + context 'when request is malformed' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = 'foobar' + end + end + let(:requesting_user) { user } + let(:expected_status) { 400 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + end + def list_issues(user:, board:, list: nil) sign_in(user) diff --git a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb index fdef9bc5638..45328482ad7 100644 --- a/spec/controllers/projects/environments/prometheus_api_controller_spec.rb +++ b/spec/controllers/projects/environments/prometheus_api_controller_spec.rb @@ -176,7 +176,7 @@ describe Projects::Environments::PrometheusApiController do def environment_params(params = {}) { id: environment.id.to_s, - namespace_id: project.namespace.name, + namespace_id: project.namespace.full_path, project_id: project.name, proxy_path: 'query', query: '1' diff --git a/spec/controllers/projects/merge_requests_controller_spec.rb b/spec/controllers/projects/merge_requests_controller_spec.rb index 9878f88a395..cc6adc0a6c6 100644 --- a/spec/controllers/projects/merge_requests_controller_spec.rb +++ b/spec/controllers/projects/merge_requests_controller_spec.rb @@ -878,7 +878,7 @@ describe Projects::MergeRequestsController do expect(control_count).to be <= 137 end - it 'has no N+1 issues for environments', :request_store, retry: 0 do + it 'has no N+1 SQL issues for environments', :request_store, retry: 0 do # First run to insert test data from lets, which does take up some 30 queries get_ci_environments_status @@ -893,18 +893,65 @@ describe Projects::MergeRequestsController do leeway = 11 expect { get_ci_environments_status }.not_to exceed_all_query_limit(control_count + leeway) end + end - def get_ci_environments_status(extra_params = {}) - params = { - namespace_id: merge_request.project.namespace.to_param, - project_id: merge_request.project, - id: merge_request.iid, - format: 'json' - } + context 'when a merge request has multiple environments with deployments' do + let(:sha) { merge_request.diff_head_sha } + let(:ref) { merge_request.source_branch } + + let!(:build) { create(:ci_build, pipeline: pipeline) } + let!(:pipeline) { create(:ci_pipeline, sha: sha, project: project) } + let!(:environment) { create(:environment, name: 'env_a', project: project) } + let!(:another_environment) { create(:environment, name: 'env_b', project: project) } + + before do + merge_request.update_head_pipeline + + create(:deployment, :succeed, environment: environment, sha: sha, ref: ref, deployable: build) + create(:deployment, :succeed, environment: another_environment, sha: sha, ref: ref, deployable: build) + end + + it 'exposes multiple environment statuses' do + get_ci_environments_status + + expect(json_response.count).to eq 2 + end + + context 'when route map is not present in the project' do + it 'does not have N+1 Gitaly requests for environments', :request_store do + expect(merge_request).to be_present + + expect { get_ci_environments_status } + .to change { Gitlab::GitalyClient.get_request_count }.by_at_most(1) + end + end - get :ci_environments_status, params: params.merge(extra_params) + context 'when there is route map present in a project' do + before do + allow_any_instance_of(EnvironmentStatus) + .to receive(:has_route_map?) + .and_return(true) + end + + it 'does not have N+1 Gitaly requests for diff files', :request_store do + expect(merge_request.merge_request_diff.merge_request_diff_files).to be_many + + expect { get_ci_environments_status } + .to change { Gitlab::GitalyClient.get_request_count }.by_at_most(1) + end end end + + def get_ci_environments_status(extra_params = {}) + params = { + namespace_id: merge_request.project.namespace.to_param, + project_id: merge_request.project, + id: merge_request.iid, + format: 'json' + } + + get :ci_environments_status, params: params.merge(extra_params) + end end describe 'GET pipeline_status.json' do diff --git a/spec/factories/services.rb b/spec/factories/services.rb index ecb481ed84a..cd1d2c33373 100644 --- a/spec/factories/services.rb +++ b/spec/factories/services.rb @@ -100,4 +100,16 @@ FactoryBot.define do type 'HipchatService' token 'test_token' end + + trait :without_properties_callback do + after(:build) do |service| + allow(service).to receive(:handle_properties) + end + + after(:create) do |service| + # we have to remove the stub because the behaviour of + # handle_properties method is tested after the creation + allow(service).to receive(:handle_properties).and_call_original + end + end end diff --git a/spec/features/merge_request/user_creates_merge_request_spec.rb b/spec/features/merge_request/user_creates_merge_request_spec.rb index d05ef2a8f12..2a70cbb2a72 100644 --- a/spec/features/merge_request/user_creates_merge_request_spec.rb +++ b/spec/features/merge_request/user_creates_merge_request_spec.rb @@ -78,7 +78,7 @@ describe "User creates a merge request", :js do click_button("Submit merge request") - expect(page).to have_content(title).and have_content("Request to merge #{user.namespace.name}:#{source_branch} into master") + expect(page).to have_content(title).and have_content("Request to merge #{user.namespace.path}:#{source_branch} into master") end end end diff --git a/spec/features/projects/fork_spec.rb b/spec/features/projects/fork_spec.rb index 7c71b4c52e0..26b94bf58b2 100644 --- a/spec/features/projects/fork_spec.rb +++ b/spec/features/projects/fork_spec.rb @@ -50,7 +50,7 @@ describe 'Project fork' do click_link('New merge request') end - expect(current_path).to have_content(/#{user.namespace.name}/i) + expect(current_path).to have_content(/#{user.namespace.path}/i) end it 'shows avatars when Gravatar is disabled' do diff --git a/spec/fixtures/api/schemas/current-board.json b/spec/fixtures/api/schemas/current-board.json new file mode 100644 index 00000000000..2ddc038e908 --- /dev/null +++ b/spec/fixtures/api/schemas/current-board.json @@ -0,0 +1,16 @@ +{ + "type": "object", + "allOf": [ + { "$ref": "board.json" }, + { + "required" : [ + "id", + "name" + ], + "properties": { + "id": { "type": "integer" }, + "name": { "type": "string" } + } + } + ] +} diff --git a/spec/fixtures/phabricator_responses/user.search.json b/spec/fixtures/phabricator_responses/user.search.json new file mode 100644 index 00000000000..f3ec653a23e --- /dev/null +++ b/spec/fixtures/phabricator_responses/user.search.json @@ -0,0 +1,62 @@ +{ + "result": { + "data": [ + { + "id": 1, + "type": "USER", + "phid": "PHID-USER-hohoho", + "fields": { + "username": "jane", + "realName": "Jane Doe", + "roles": [ + "admin", + "verified", + "approved", + "activated" + ], + "dateCreated": 1405970599, + "dateModified": 1406705963, + "policy": { + "view": "public", + "edit": "no-one" + } + }, + "attachments": {} + }, + { + "id": 2, + "type": "USER", + "phid": "PHID-USER-hihihi", + "fields": { + "username": "john", + "realName": "John Doe", + "roles": [ + "admin", + "verified", + "approved", + "activated" + ], + "dateCreated": 1403609184, + "dateModified": 1559138722, + "policy": { + "view": "public", + "edit": "no-one" + } + }, + "attachments": {} + } + ], + "maps": {}, + "query": { + "queryKey": null + }, + "cursor": { + "limit": "100", + "after": null, + "before": null, + "order": null + } + }, + "error_code": null, + "error_info": null +} diff --git a/spec/frontend/filterable_list_spec.js b/spec/frontend/filterable_list_spec.js new file mode 100644 index 00000000000..67d18611661 --- /dev/null +++ b/spec/frontend/filterable_list_spec.js @@ -0,0 +1,53 @@ +import FilterableList from '~/filterable_list'; +import { getJSONFixture, setHTMLFixture } from './helpers/fixtures'; + +describe('FilterableList', () => { + let List; + let form; + let filter; + let holder; + + beforeEach(() => { + setHTMLFixture(` + <form id="project-filter-form"> + <input name="name" class="js-projects-list-filter" /> + </div> + <div class="js-projects-list-holder"></div> + `); + getJSONFixture('static/projects.json'); + form = document.querySelector('form#project-filter-form'); + filter = document.querySelector('.js-projects-list-filter'); + holder = document.querySelector('.js-projects-list-holder'); + List = new FilterableList(form, filter, holder); + }); + + it('processes input parameters', () => { + expect(List.filterForm).toEqual(form); + expect(List.listFilterElement).toEqual(filter); + expect(List.listHolderElement).toEqual(holder); + }); + + describe('getPagePath', () => { + it('returns properly constructed base endpoint', () => { + List.filterForm.action = '/foo/bar/'; + List.listFilterElement.value = 'blah'; + + expect(List.getPagePath()).toEqual('/foo/bar/?name=blah'); + }); + + it('properly appends custom parameters to existing URL', () => { + List.filterForm.action = '/foo/bar?alpha=beta'; + List.listFilterElement.value = 'blah'; + + expect(List.getPagePath()).toEqual('/foo/bar?alpha=beta&name=blah'); + }); + }); + + describe('getFilterEndpoint', () => { + it('returns getPagePath by default', () => { + jest.spyOn(List, 'getPagePath').mockReturnValue('blah/blah/foo'); + + expect(List.getFilterEndpoint()).toEqual(List.getPagePath()); + }); + }); +}); diff --git a/spec/frontend/projects/projects_filterable_list_spec.js b/spec/frontend/projects/projects_filterable_list_spec.js new file mode 100644 index 00000000000..e756fb3ab56 --- /dev/null +++ b/spec/frontend/projects/projects_filterable_list_spec.js @@ -0,0 +1,31 @@ +import ProjectsFilterableList from '~/projects/projects_filterable_list'; +import { getJSONFixture, setHTMLFixture } from '../helpers/fixtures'; + +describe('ProjectsFilterableList', () => { + let List; + let form; + let filter; + let holder; + + beforeEach(() => { + setHTMLFixture(` + <form id="project-filter-form"> + <input name="name" class="js-projects-list-filter" /> + </div> + <div class="js-projects-list-holder"></div> + `); + getJSONFixture('static/projects.json'); + form = document.querySelector('form#project-filter-form'); + filter = document.querySelector('.js-projects-list-filter'); + holder = document.querySelector('.js-projects-list-holder'); + List = new ProjectsFilterableList(form, filter, holder); + }); + + describe('getFilterEndpoint', () => { + it('updates converts getPagePath for projects', () => { + jest.spyOn(List, 'getPagePath').mockReturnValue('blah/projects?'); + + expect(List.getFilterEndpoint()).toEqual('blah/projects.json?'); + }); + }); +}); diff --git a/spec/helpers/avatars_helper_spec.rb b/spec/helpers/avatars_helper_spec.rb index aa0442ab847..94998d302f9 100644 --- a/spec/helpers/avatars_helper_spec.rb +++ b/spec/helpers/avatars_helper_spec.rb @@ -16,7 +16,7 @@ describe AvatarsHelper do shared_examples 'resource with a custom avatar' do |source_type| it 'returns a custom avatar image' do expect(public_send("#{source_type}_icon", *helper_args)) - .to eq "<img src=\"#{resource.avatar.url}\" alt=\"Banana sample\" />" + .to eq "<img src=\"#{resource.avatar.url}\" />" end end diff --git a/spec/helpers/boards_helper_spec.rb b/spec/helpers/boards_helper_spec.rb index b22947911b9..f014537eb54 100644 --- a/spec/helpers/boards_helper_spec.rb +++ b/spec/helpers/boards_helper_spec.rb @@ -1,10 +1,12 @@ require 'spec_helper' describe BoardsHelper do + set(:project) { create(:project) } + describe '#build_issue_link_base' do context 'project board' do it 'returns correct path for project board' do - @project = create(:project) + @project = project @board = create(:board, project: @project) expect(build_issue_link_base).to eq("/#{@project.namespace.path}/#{@project.path}/issues") @@ -31,7 +33,6 @@ describe BoardsHelper do describe '#board_data' do let(:user) { create(:user) } - let(:project) { create(:project) } let(:board) { create(:board, project: project) } before do @@ -46,4 +47,15 @@ describe BoardsHelper do expect(helper.board_data[:lists_endpoint]).to eq(board_lists_path(board)) end end + + describe '#current_board_json' do + let(:board_json) { helper.current_board_json } + + it 'can serialise with a basic set of attributes' do + board = create(:board, project: project) + assign(:board, board) + + expect(board_json).to match_schema('current-board') + end + end end diff --git a/spec/helpers/emails_helper_spec.rb b/spec/helpers/emails_helper_spec.rb index e6aacb5b92b..d25f0c6de4a 100644 --- a/spec/helpers/emails_helper_spec.rb +++ b/spec/helpers/emails_helper_spec.rb @@ -103,7 +103,7 @@ describe EmailsHelper do appearance = create :appearance, header_logo: fixture_file_upload('spec/fixtures/dk.png') expect(header_logo).to eq( - %{<img style="height: 50px" src="/uploads/-/system/appearance/header_logo/#{appearance.id}/dk.png" alt="Dk" />} + %{<img style="height: 50px" src="/uploads/-/system/appearance/header_logo/#{appearance.id}/dk.png" />} ) end end diff --git a/spec/helpers/markup_helper_spec.rb b/spec/helpers/markup_helper_spec.rb index 6c6410cee9b..f6e1720e113 100644 --- a/spec/helpers/markup_helper_spec.rb +++ b/spec/helpers/markup_helper_spec.rb @@ -268,7 +268,7 @@ describe MarkupHelper do end end - describe 'markup' do + describe '#markup' do let(:content) { 'Noël' } it 'preserves encoding' do @@ -302,6 +302,77 @@ describe MarkupHelper do end end + describe '#markup_unsafe' do + subject { helper.markup_unsafe(file_name, text, context) } + + let(:file_name) { 'foo.bar' } + let(:text) { 'Noël' } + let(:project_base) { build(:project, :repository) } + let(:context) { { project: project_base } } + + context 'when text is missing' do + let(:text) { nil } + + it 'returns an empty string' do + is_expected.to eq('') + end + end + + context 'when file is a markdown file' do + let(:file_name) { 'foo.md' } + + it 'returns html (rendered by Banzai)' do + expected_html = '<p data-sourcepos="1:1-1:5" dir="auto">Noël</p>' + + expect(Banzai).to receive(:render).with(text, context) { expected_html } + + is_expected.to eq(expected_html) + end + + context 'when renderer returns an error' do + before do + allow(Banzai).to receive(:render).and_raise("An error") + end + + it 'returns html (rendered by ActionView:TextHelper)' do + is_expected.to eq('<p>Noël</p>') + end + end + end + + context 'when file is asciidoc file' do + let(:file_name) { 'foo.adoc' } + + it 'returns html (rendered by Gitlab::Asciidoc)' do + expected_html = "<div>\n<p>Noël</p>\n</div>" + + expect(Gitlab::Asciidoc).to receive(:render).with(text, context) { expected_html } + + is_expected.to eq(expected_html) + end + end + + context 'when file is a regular text file' do + let(:file_name) { 'foo.txt' } + + it 'returns html (rendered by ActionView::TagHelper)' do + is_expected.to eq('<pre class="plain-readme">Noël</pre>') + end + end + + context 'when file has an unknown type' do + let(:file_name) { 'foo' } + + it 'returns html (rendered by Gitlab::OtherMarkup)' do + expected_html = 'Noël' + + expect(Gitlab::OtherMarkup).to receive(:render).with(file_name, text, context) { expected_html } + + is_expected.to eq(expected_html) + end + end + end + describe '#first_line_in_markdown' do shared_examples_for 'common markdown examples' do let(:project_base) { build(:project, :repository) } diff --git a/spec/helpers/sorting_helper_spec.rb b/spec/helpers/sorting_helper_spec.rb index f405268d198..5397a47b3dd 100644 --- a/spec/helpers/sorting_helper_spec.rb +++ b/spec/helpers/sorting_helper_spec.rb @@ -4,6 +4,11 @@ require 'spec_helper' describe SortingHelper do include ApplicationHelper include IconsHelper + include ExploreHelper + + def set_sorting_url(option) + allow(self).to receive(:request).and_return(double(path: 'http://test.com', query_parameters: { label_name: option })) + end describe '#issuable_sort_option_title' do it 'returns correct title for issuable_sort_option_overrides key' do @@ -21,7 +26,7 @@ describe SortingHelper do describe '#issuable_sort_direction_button' do before do - allow(self).to receive(:request).and_return(double(path: 'http://test.com', query_parameters: { label_name: 'test_label' })) + set_sorting_url 'test_label' end it 'keeps label filter param' do @@ -44,4 +49,145 @@ describe SortingHelper do expect(issuable_sort_direction_button('due_date')).to include('sort-lowest') end end + + def stub_controller_path(value) + allow(helper.controller).to receive(:controller_path).and_return(value) + end + + def project_common_options + { + sort_value_latest_activity => sort_title_latest_activity, + sort_value_recently_created => sort_title_created_date, + sort_value_name => sort_title_name, + sort_value_stars_desc => sort_title_stars + } + end + + describe 'with `admin/projects` controller' do + before do + stub_controller_path 'admin/projects' + end + + describe '#projects_sort_options_hash' do + it 'returns a hash of available sorting options' do + admin_options = project_common_options.merge({ + sort_value_oldest_activity => sort_title_oldest_activity, + sort_value_oldest_created => sort_title_oldest_created, + sort_value_recently_created => sort_title_recently_created, + sort_value_stars_desc => sort_title_most_stars, + sort_value_largest_repo => sort_title_largest_repo + }) + + expect(projects_sort_options_hash).to eq(admin_options) + end + end + end + + describe 'with `projects` controller' do + before do + stub_controller_path 'projects' + end + + describe '#projects_sort_options_hash' do + it 'returns a hash of available sorting options' do + expect(projects_sort_options_hash).to include(project_common_options) + end + end + + describe '#projects_reverse_sort_options_hash' do + context 'returns a reversed hash of available sorting options' do + using RSpec::Parameterized::TableSyntax + + where(:sort_key, :reverse_sort_title) do + sort_value_latest_activity | sort_value_oldest_activity + sort_value_recently_created | sort_value_oldest_created + sort_value_name | sort_value_name_desc + sort_value_stars_desc | sort_value_stars_asc + sort_value_oldest_activity | sort_value_latest_activity + sort_value_oldest_created | sort_value_recently_created + sort_value_name_desc | sort_value_name + sort_value_stars_asc | sort_value_stars_desc + end + + with_them do + it do + reverse_hash = projects_reverse_sort_options_hash + + expect(reverse_hash).to include(sort_key) + expect(reverse_hash[sort_key]).to eq(reverse_sort_title) + end + end + end + end + + describe '#project_sort_direction_button' do + context 'returns the correct icon for each sort option' do + using RSpec::Parameterized::TableSyntax + + sort_lowest_icon = 'sort-lowest' + sort_highest_icon = 'sort-highest' + + where(:selected_sort, :icon) do + sort_value_latest_activity | sort_highest_icon + sort_value_recently_created | sort_highest_icon + sort_value_name_desc | sort_highest_icon + sort_value_stars_desc | sort_highest_icon + sort_value_oldest_activity | sort_lowest_icon + sort_value_oldest_created | sort_lowest_icon + sort_value_name | sort_lowest_icon + sort_value_stars_asc | sort_lowest_icon + end + + with_them do + it do + set_sorting_url selected_sort + + expect(project_sort_direction_button(selected_sort)).to include(icon) + end + end + end + + it 'returns the correct link to reverse the current sort option' do + sort_options_links = projects_reverse_sort_options_hash + + sort_options_links.each do |selected_sort, reverse_sort| + set_sorting_url selected_sort + + expect(project_sort_direction_button(selected_sort)).to include(reverse_sort) + end + end + end + + describe '#projects_sort_option_titles' do + it 'returns a hash of titles for the sorting options' do + options = project_common_options.merge({ + sort_value_oldest_activity => sort_title_latest_activity, + sort_value_oldest_created => sort_title_created_date, + sort_value_name_desc => sort_title_name, + sort_value_stars_asc => sort_title_stars + }) + + expect(projects_sort_option_titles).to eq(options) + end + end + + describe 'with project_list_filter_bar off' do + before do + stub_feature_flags(project_list_filter_bar: false) + end + + describe '#projects_sort_options_hash' do + it 'returns a hash of available sorting options' do + options = project_common_options.merge({ + sort_value_oldest_activity => sort_title_oldest_activity, + sort_value_oldest_created => sort_title_oldest_created, + sort_value_recently_created => sort_title_recently_created, + sort_value_stars_desc => sort_title_most_stars + }) + + expect(projects_sort_options_hash).to eq(options) + end + end + end + end end diff --git a/spec/javascripts/helpers/vuex_action_helper.js b/spec/javascripts/helpers/vuex_action_helper.js index 88652202a8e..c5de31a4138 100644 --- a/spec/javascripts/helpers/vuex_action_helper.js +++ b/spec/javascripts/helpers/vuex_action_helper.js @@ -89,9 +89,7 @@ export default ( payload, ); - return new Promise(resolve => { - setImmediate(resolve); - }) + return new Promise(setImmediate) .then(() => result) .catch(error => { validateResults(); diff --git a/spec/javascripts/monitoring/charts/single_stat_spec.js b/spec/javascripts/monitoring/charts/single_stat_spec.js index 12b73002f97..127a4a7955a 100644 --- a/spec/javascripts/monitoring/charts/single_stat_spec.js +++ b/spec/javascripts/monitoring/charts/single_stat_spec.js @@ -1,5 +1,6 @@ import { shallowMount } from '@vue/test-utils'; import SingleStatChart from '~/monitoring/components/charts/single_stat.vue'; +import { graphDataPrometheusQuery } from '../mock_data'; describe('Single Stat Chart component', () => { let singleStatChart; @@ -7,9 +8,7 @@ describe('Single Stat Chart component', () => { beforeEach(() => { singleStatChart = shallowMount(SingleStatChart, { propsData: { - title: 'Time to render', - value: 1, - unit: 'sec', + graphData: graphDataPrometheusQuery, }, }); }); @@ -19,9 +18,9 @@ describe('Single Stat Chart component', () => { }); describe('computed', () => { - describe('valueWithUnit', () => { + describe('engineeringNotation', () => { it('should interpolate the value and unit props', () => { - expect(singleStatChart.vm.valueWithUnit).toBe('1sec'); + expect(singleStatChart.vm.engineeringNotation).toBe('91MB'); }); }); }); diff --git a/spec/javascripts/monitoring/mock_data.js b/spec/javascripts/monitoring/mock_data.js index 7bbb215475a..85e660d3925 100644 --- a/spec/javascripts/monitoring/mock_data.js +++ b/spec/javascripts/monitoring/mock_data.js @@ -935,3 +935,75 @@ export const dashboardGitResponse = [ default: false, }, ]; + +export const graphDataPrometheusQuery = { + title: 'Super Chart A2', + type: 'single-stat', + weight: 2, + metrics: [ + { + id: 'metric_a1', + metric_id: 2, + query: 'max(go_memstats_alloc_bytes{job="prometheus"}) by (job) /1024/1024', + unit: 'MB', + label: 'Total Consumption', + prometheus_endpoint_path: + '/root/kubernetes-gke-project/environments/35/prometheus/api/v1/query?query=max%28go_memstats_alloc_bytes%7Bjob%3D%22prometheus%22%7D%29+by+%28job%29+%2F1024%2F1024', + }, + ], + queries: [ + { + metricId: null, + id: 'metric_a1', + metric_id: 2, + query: 'max(go_memstats_alloc_bytes{job="prometheus"}) by (job) /1024/1024', + unit: 'MB', + label: 'Total Consumption', + prometheus_endpoint_path: + '/root/kubernetes-gke-project/environments/35/prometheus/api/v1/query?query=max%28go_memstats_alloc_bytes%7Bjob%3D%22prometheus%22%7D%29+by+%28job%29+%2F1024%2F1024', + result: [ + { + metric: { job: 'prometheus' }, + value: ['2019-06-26T21:03:20.881Z', 91], + }, + ], + }, + ], +}; + +export const graphDataPrometheusQueryRange = { + title: 'Super Chart A1', + type: 'area', + weight: 2, + metrics: [ + { + id: 'metric_a1', + metric_id: 2, + query_range: + 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024', + unit: 'MB', + label: 'Total Consumption', + prometheus_endpoint_path: + '/root/kubernetes-gke-project/environments/35/prometheus/api/v1/query?query=max%28go_memstats_alloc_bytes%7Bjob%3D%22prometheus%22%7D%29+by+%28job%29+%2F1024%2F1024', + }, + ], + queries: [ + { + metricId: null, + id: 'metric_a1', + metric_id: 2, + query_range: + 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024', + unit: 'MB', + label: 'Total Consumption', + prometheus_endpoint_path: + '/root/kubernetes-gke-project/environments/35/prometheus/api/v1/query?query=max%28go_memstats_alloc_bytes%7Bjob%3D%22prometheus%22%7D%29+by+%28job%29+%2F1024%2F1024', + result: [ + { + metric: {}, + values: [[1495700554.925, '8.0390625'], [1495700614.925, '8.0390625']], + }, + ], + }, + ], +}; diff --git a/spec/javascripts/monitoring/store/mutations_spec.js b/spec/javascripts/monitoring/store/mutations_spec.js index 91580366531..43776b1b7f2 100644 --- a/spec/javascripts/monitoring/store/mutations_spec.js +++ b/spec/javascripts/monitoring/store/mutations_spec.js @@ -115,12 +115,14 @@ describe('Monitoring mutations', () => { environmentsEndpoint: 'environments.json', deploymentsEndpoint: 'deployments.json', dashboardEndpoint: 'dashboard.json', + projectPath: '/gitlab-org/gitlab-ce', }); expect(stateCopy.metricsEndpoint).toEqual('additional_metrics.json'); expect(stateCopy.environmentsEndpoint).toEqual('environments.json'); expect(stateCopy.deploymentsEndpoint).toEqual('deployments.json'); expect(stateCopy.dashboardEndpoint).toEqual('dashboard.json'); + expect(stateCopy.projectPath).toEqual('/gitlab-org/gitlab-ce'); }); }); diff --git a/spec/javascripts/monitoring/utils_spec.js b/spec/javascripts/monitoring/utils_spec.js index e3c455d1686..5570d57b8b2 100644 --- a/spec/javascripts/monitoring/utils_spec.js +++ b/spec/javascripts/monitoring/utils_spec.js @@ -1,5 +1,6 @@ -import { getTimeDiff } from '~/monitoring/utils'; +import { getTimeDiff, graphDataValidatorForValues } from '~/monitoring/utils'; import { timeWindows } from '~/monitoring/constants'; +import { graphDataPrometheusQuery, graphDataPrometheusQueryRange } from './mock_data'; describe('getTimeDiff', () => { it('defaults to an 8 hour (28800s) difference', () => { @@ -27,3 +28,27 @@ describe('getTimeDiff', () => { }); }); }); + +describe('graphDataValidatorForValues', () => { + /* + * When dealing with a metric using the query format, e.g. + * query: 'max(go_memstats_alloc_bytes{job="prometheus"}) by (job) /1024/1024' + * the validator will look for the `value` key instead of `values` + */ + it('validates data with the query format', () => { + const validGraphData = graphDataValidatorForValues(true, graphDataPrometheusQuery); + + expect(validGraphData).toBe(true); + }); + + /* + * When dealing with a metric using the query?range format, e.g. + * query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) /1024/1024/1024', + * the validator will look for the `values` key instead of `value` + */ + it('validates data with the query_range format', () => { + const validGraphData = graphDataValidatorForValues(false, graphDataPrometheusQueryRange); + + expect(validGraphData).toBe(true); + }); +}); diff --git a/spec/javascripts/notes/stores/actions_spec.js b/spec/javascripts/notes/stores/actions_spec.js index c97ff5236ec..c461c28a37b 100644 --- a/spec/javascripts/notes/stores/actions_spec.js +++ b/spec/javascripts/notes/stores/actions_spec.js @@ -18,6 +18,8 @@ import { noteableDataMock, individualNote, } from '../mock_data'; +import AxiosMockAdapter from 'axios-mock-adapter'; +import axios from '~/lib/utils/axios_utils'; const TEST_ERROR_MESSAGE = 'Test error message'; @@ -335,28 +337,24 @@ describe('Actions Notes Store', () => { }); describe('deleteNote', () => { - const interceptor = (request, next) => { - next( - request.respondWith(JSON.stringify({}), { - status: 200, - }), - ); - }; + const endpoint = `${TEST_HOST}/note`; + let axiosMock; beforeEach(() => { - Vue.http.interceptors.push(interceptor); + axiosMock = new AxiosMockAdapter(axios); + axiosMock.onDelete(endpoint).replyOnce(200, {}); $('body').attr('data-page', ''); }); afterEach(() => { - Vue.http.interceptors = _.without(Vue.http.interceptors, interceptor); + axiosMock.restore(); $('body').attr('data-page', ''); }); it('commits DELETE_NOTE and dispatches updateMergeRequestWidget', done => { - const note = { path: `${gl.TEST_HOST}`, id: 1 }; + const note = { path: endpoint, id: 1 }; testAction( actions.deleteNote, @@ -381,7 +379,7 @@ describe('Actions Notes Store', () => { }); it('dispatches removeDiscussionsFromDiff on merge request page', done => { - const note = { path: `${gl.TEST_HOST}`, id: 1 }; + const note = { path: endpoint, id: 1 }; $('body').attr('data-page', 'projects:merge_requests:show'); diff --git a/spec/javascripts/registry/components/collapsible_container_spec.js b/spec/javascripts/registry/components/collapsible_container_spec.js index 55017b3e26b..2a5d8dd11da 100644 --- a/spec/javascripts/registry/components/collapsible_container_spec.js +++ b/spec/javascripts/registry/components/collapsible_container_spec.js @@ -77,7 +77,7 @@ describe('collapsible registry container', () => { spyOn(vm, 'deleteItem').and.returnValue(Promise.resolve()); Vue.nextTick(() => { - document.querySelector('#confirm-repo-deletion-modal .btn-danger').click(); + document.querySelector(`#${vm.modalId} .btn-danger`).click(); expect(vm.deleteItem).toHaveBeenCalledWith(vm.repo); done(); diff --git a/spec/javascripts/registry/components/table_registry_spec.js b/spec/javascripts/registry/components/table_registry_spec.js index 6a0b16f592e..31ac970378e 100644 --- a/spec/javascripts/registry/components/table_registry_spec.js +++ b/spec/javascripts/registry/components/table_registry_spec.js @@ -51,7 +51,7 @@ describe('table registry', () => { spyOn(vm, 'deleteItem').and.returnValue(Promise.resolve()); Vue.nextTick(() => { - document.querySelector('#confirm-image-deletion-modal .btn-danger').click(); + document.querySelector(`#${vm.modalId} .btn-danger`).click(); expect(vm.deleteItem).toHaveBeenCalledWith(firstImage); expect(vm.itemToBeDeleted).toBeNull(); diff --git a/spec/javascripts/test_bundle.js b/spec/javascripts/test_bundle.js index 50741e249ca..ce453d7c483 100644 --- a/spec/javascripts/test_bundle.js +++ b/spec/javascripts/test_bundle.js @@ -3,6 +3,7 @@ */ import $ from 'jquery'; +import 'core-js/features/set-immediate'; import 'vendor/jasmine-jquery'; import '~/commons'; import Vue from 'vue'; diff --git a/spec/lib/api/helpers/pagination_spec.rb b/spec/lib/api/helpers/pagination_spec.rb index c788da55cd2..b0a00392957 100644 --- a/spec/lib/api/helpers/pagination_spec.rb +++ b/spec/lib/api/helpers/pagination_spec.rb @@ -114,7 +114,7 @@ describe API::Helpers::Pagination do expect(paginated_relation.order_values).to be_present expect(paginated_relation.order_values.size).to eq(1) expect(paginated_relation.order_values.first).to be_descending - expect(paginated_relation.order_values.first.expr.name).to eq :id + expect(paginated_relation.order_values.first.expr.name).to eq 'id' end end @@ -151,9 +151,9 @@ describe API::Helpers::Pagination do expect(paginated_relation.order_values).to be_present expect(paginated_relation.order_values.size).to eq(2) expect(paginated_relation.order_values.first).to be_descending - expect(paginated_relation.order_values.first.expr.name).to eq :name + expect(paginated_relation.order_values.first.expr.name).to eq 'name' expect(paginated_relation.order_values.second).to be_descending - expect(paginated_relation.order_values.second.expr.name).to eq :id + expect(paginated_relation.order_values.second.expr.name).to eq 'id' end it 'returns the right records (first page)' do @@ -341,7 +341,7 @@ describe API::Helpers::Pagination do expect(resource.order_values).to be_empty expect(paginated_relation.order_values).to be_present expect(paginated_relation.order_values.first).to be_ascending - expect(paginated_relation.order_values.first.expr.name).to eq :id + expect(paginated_relation.order_values.first.expr.name).to eq 'id' end it 'is present it does not add anything' do @@ -349,7 +349,7 @@ describe API::Helpers::Pagination do expect(paginated_relation.order_values).to be_present expect(paginated_relation.order_values.first).to be_descending - expect(paginated_relation.order_values.first.expr.name).to eq :created_at + expect(paginated_relation.order_values.first.expr.name).to eq 'created_at' end end end diff --git a/spec/lib/banzai/renderer_spec.rb b/spec/lib/banzai/renderer_spec.rb index aa828e2f0e9..a099f7482c1 100644 --- a/spec/lib/banzai/renderer_spec.rb +++ b/spec/lib/banzai/renderer_spec.rb @@ -19,6 +19,24 @@ describe Banzai::Renderer do object end + describe '#cache_collection_render' do + let(:merge_request) { fake_object(fresh: true) } + let(:context) { { cache_key: [merge_request, 'field'], rendered: merge_request.field_html } } + + context 'when an item has a rendered field' do + before do + allow(merge_request).to receive(:field).and_return('This is the field') + allow(merge_request).to receive(:field_html).and_return('This is the field') + end + + it 'does not touch redis if the field is in the cache' do + expect(Rails).not_to receive(:cache) + + described_class.cache_collection_render([{ text: merge_request.field, context: context }]) + end + end + end + describe '#render_field' do let(:renderer) { described_class } diff --git a/spec/lib/gitlab/asciidoc_spec.rb b/spec/lib/gitlab/asciidoc_spec.rb index 8f2434acd26..ff002acbd35 100644 --- a/spec/lib/gitlab/asciidoc_spec.rb +++ b/spec/lib/gitlab/asciidoc_spec.rb @@ -45,28 +45,117 @@ module Gitlab end context "XSS" do - links = { - 'links' => { + items = { + 'link with extra attribute' => { input: 'link:mylink"onmouseover="alert(1)[Click Here]', output: "<div>\n<p><a href=\"mylink\">Click Here</a></p>\n</div>" }, - 'images' => { + 'link with unsafe scheme' => { + input: 'link:data://danger[Click Here]', + output: "<div>\n<p><a>Click Here</a></p>\n</div>" + }, + 'image with onerror' => { input: 'image:https://localhost.com/image.png[Alt text" onerror="alert(7)]', output: "<div>\n<p><span><img src=\"https://localhost.com/image.png\" alt='Alt text\" onerror=\"alert(7)'></span></p>\n</div>" }, - 'pre' => { + 'fenced code with inline script' => { input: '```mypre"><script>alert(3)</script>', output: "<div>\n<div>\n<pre class=\"code highlight js-syntax-highlight plaintext\" lang=\"plaintext\" v-pre=\"true\"><code><span id=\"LC1\" class=\"line\" lang=\"plaintext\">\"></span></code></pre>\n</div>\n</div>" } } - links.each do |name, data| + items.each do |name, data| it "does not convert dangerous #{name} into HTML" do expect(render(data[:input], context)).to include(data[:output]) end end end + context 'with admonition' do + it 'preserves classes' do + input = <<~ADOC + NOTE: An admonition paragraph, like this note, grabs the reader’s attention. + ADOC + + output = <<~HTML + <div class="admonitionblock"> + <table> + <tr> + <td class="icon"> + <i class="fa icon-note" title="Note"></i> + </td> + <td> + An admonition paragraph, like this note, grabs the reader’s attention. + </td> + </tr> + </table> + </div> + HTML + + expect(render(input, context)).to include(output.strip) + end + end + + context 'with checklist' do + it 'preserves classes' do + input = <<~ADOC + * [x] checked + * [ ] not checked + ADOC + + output = <<~HTML + <div> + <ul class="checklist"> + <li> + <p><i class="fa fa-check-square-o"></i> checked</p> + </li> + <li> + <p><i class="fa fa-square-o"></i> not checked</p> + </li> + </ul> + </div> + HTML + + expect(render(input, context)).to include(output.strip) + end + end + + context 'with marks' do + it 'preserves classes' do + input = <<~ADOC + Werewolves are allergic to #cassia cinnamon#. + + Did the werewolves read the [.small]#small print#? + + Where did all the [.underline.small]#cores# run off to? + + We need [.line-through]#ten# make that twenty VMs. + + [.big]##O##nce upon an infinite loop. + ADOC + + output = <<~HTML + <div> + <p>Werewolves are allergic to <mark>cassia cinnamon</mark>.</p> + </div> + <div> + <p>Did the werewolves read the <span class="small">small print</span>?</p> + </div> + <div> + <p>Where did all the <span class="underline small">cores</span> run off to?</p> + </div> + <div> + <p>We need <span class="line-through">ten</span> make that twenty VMs.</p> + </div> + <div> + <p><span class="big">O</span>nce upon an infinite loop.</p> + </div> + HTML + + expect(render(input, context)).to include(output.strip) + end + end + context 'with fenced block' do it 'highlights syntax' do input = <<~ADOC diff --git a/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb new file mode 100644 index 00000000000..5938ecca459 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_user_namespace_names_spec.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BackgroundMigration::FixUserNamespaceNames, :migration, schema: 20190620112608 do + let(:namespaces) { table(:namespaces) } + let(:users) { table(:users) } + let(:user) { users.create(name: "The user's full name", projects_limit: 10, username: 'not-null', email: '1') } + + context 'updating the namespace names' do + it 'updates a user namespace within range' do + user2 = users.create(name: "Other user's full name", projects_limit: 10, username: 'also-not-null', email: '2') + user_namespace1 = namespaces.create( + id: 2, + owner_id: user.id, + name: "Should be the user's name", + path: user.username + ) + user_namespace2 = namespaces.create( + id: 3, + owner_id: user2.id, + name: "Should also be the user's name", + path: user.username + ) + + described_class.new.perform(1, 5) + + expect(user_namespace1.reload.name).to eq("The user's full name") + expect(user_namespace2.reload.name).to eq("Other user's full name") + end + + it 'does not update namespaces out of range' do + user_namespace = namespaces.create( + id: 6, + owner_id: user.id, + name: "Should be the user's name", + path: user.username + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { user_namespace.reload.name } + end + + it 'does not update groups owned by the users' do + user_group = namespaces.create( + id: 2, + owner_id: user.id, + name: 'A group name', + path: 'the-path', + type: 'Group' + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { user_group.reload.name } + end + end + + context 'namespace route names' do + let(:routes) { table(:routes) } + let(:namespace) do + namespaces.create( + id: 2, + owner_id: user.id, + name: "Will be updated to the user's name", + path: user.username + ) + end + + it "updates the route name if it didn't match the namespace" do + route = routes.create(path: namespace.path, name: 'Incorrect name', source_type: 'Namespace', source_id: namespace.id) + + described_class.new.perform(1, 5) + + expect(route.reload.name).to eq("The user's full name") + end + + it 'updates the route name if it was nil match the namespace' do + route = routes.create(path: namespace.path, name: nil, source_type: 'Namespace', source_id: namespace.id) + + described_class.new.perform(1, 5) + + expect(route.reload.name).to eq("The user's full name") + end + + it "doesn't update group routes" do + route = routes.create(path: 'group-path', name: 'Group name', source_type: 'Group', source_id: namespace.id) + + expect { described_class.new.perform(1, 5) } + .not_to change { route.reload.name } + end + + it "doesn't touch routes for namespaces out of range" do + user_namespace = namespaces.create( + id: 6, + owner_id: user.id, + name: "Should be the user's name", + path: user.username + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { user_namespace.reload.name } + end + end +end diff --git a/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb b/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb new file mode 100644 index 00000000000..d1d6d8411d1 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_user_project_route_names_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BackgroundMigration::FixUserProjectRouteNames, :migration, schema: 20190620112608 do + let(:namespaces) { table(:namespaces) } + let(:users) { table(:users) } + let(:routes) { table(:routes) } + let(:projects) { table(:projects) } + + let(:user) { users.create(name: "The user's full name", projects_limit: 10, username: 'not-null', email: '1') } + + let(:namespace) do + namespaces.create( + owner_id: user.id, + name: "Should eventually be the user's name", + path: user.username + ) + end + + let(:project) do + projects.create(namespace_id: namespace.id, name: 'Project Name') + end + + it "updates the route for a project if it did not match the user's name" do + route = routes.create( + id: 1, + path: "#{user.username}/#{project.path}", + source_id: project.id, + source_type: 'Project', + name: 'Completely wrong' + ) + + described_class.new.perform(1, 5) + + expect(route.reload.name).to eq("The user's full name / Project Name") + end + + it 'updates the route for a project if the name was nil' do + route = routes.create( + id: 1, + path: "#{user.username}/#{project.path}", + source_id: project.id, + source_type: 'Project', + name: nil + ) + + described_class.new.perform(1, 5) + + expect(route.reload.name).to eq("The user's full name / Project Name") + end + + it 'does not update routes that were are out of the range' do + route = routes.create( + id: 6, + path: "#{user.username}/#{project.path}", + source_id: project.id, + source_type: 'Project', + name: 'Completely wrong' + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { route.reload.name } + end + + it 'does not update routes for projects in groups owned by the user' do + group = namespaces.create( + owner_id: user.id, + name: 'A group', + path: 'a-path', + type: '' + ) + project = projects.create(namespace_id: group.id, name: 'Project Name') + route = routes.create( + id: 1, + path: "#{group.path}/#{project.path}", + source_id: project.id, + source_type: 'Project', + name: 'Completely wrong' + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { route.reload.name } + end + + it 'does not update routes for namespaces' do + route = routes.create( + id: 1, + path: namespace.path, + source_id: namespace.id, + source_type: 'Namespace', + name: 'Completely wrong' + ) + + expect { described_class.new.perform(1, 5) } + .not_to change { route.reload.name } + end +end diff --git a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb b/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb index 4a81a37d341..ad4fa4fe03a 100644 --- a/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_merge_request_assignees_table_spec.rb @@ -27,14 +27,19 @@ describe Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable, :migra merge_requests.create(params) end + before do + create_merge_request(2, assignee_id: user.id) + create_merge_request(3, assignee_id: user_2.id) + create_merge_request(4, assignee_id: user_3.id) + + # Test filtering MRs without assignees + create_merge_request(5, assignee_id: nil) + # Test filtering already migrated row + merge_request_assignees.create!(merge_request_id: 2, user_id: user_3.id) + end + describe '#perform' do it 'creates merge_request_assignees rows according to merge_requests' do - create_merge_request(2, assignee_id: user.id) - create_merge_request(3, assignee_id: user_2.id) - create_merge_request(4, assignee_id: user_3.id) - # Test filtering already migrated row - merge_request_assignees.create!(merge_request_id: 2, user_id: user_3.id) - subject.perform(1, 4) rows = merge_request_assignees.order(:id).map { |row| row.attributes.slice('merge_request_id', 'user_id') } @@ -53,4 +58,13 @@ describe Gitlab::BackgroundMigration::PopulateMergeRequestAssigneesTable, :migra end end end + + describe '#perform_all_sync' do + it 'executes peform for all merge requests in batches' do + expect(subject).to receive(:perform).with(2, 4).ordered + expect(subject).to receive(:perform).with(5, 5).ordered + + subject.perform_all_sync(batch_size: 3) + end + end end diff --git a/spec/lib/gitlab/batch_pop_queueing_spec.rb b/spec/lib/gitlab/batch_pop_queueing_spec.rb new file mode 100644 index 00000000000..28984d52024 --- /dev/null +++ b/spec/lib/gitlab/batch_pop_queueing_spec.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BatchPopQueueing do + include ExclusiveLeaseHelpers + using RSpec::Parameterized::TableSyntax + + describe '#initialize' do + where(:namespace, :queue_id, :expect_error, :error_type) do + 'feature' | '1' | false | nil + :feature | '1' | false | nil + nil | '1' | true | NoMethodError + 'feature' | nil | true | NoMethodError + '' | '1' | true | ArgumentError + 'feature' | '' | true | ArgumentError + 'feature' | 1 | true | NoMethodError + end + + with_them do + it do + if expect_error + expect { described_class.new(namespace, queue_id) }.to raise_error(error_type) + else + expect { described_class.new(namespace, queue_id) }.not_to raise_error + end + end + end + end + + describe '#safe_execute', :clean_gitlab_redis_queues do + subject { queue.safe_execute(new_items, lock_timeout: lock_timeout) } + + let(:queue) { described_class.new(namespace, queue_id) } + let(:namespace) { 'feature' } + let(:queue_id) { '1' } + let(:lock_timeout) { 10.minutes } + let(:new_items) { %w[A B] } + let(:lock_key) { queue.send(:lock_key) } + let(:queue_key) { queue.send(:queue_key) } + + it 'enqueues new items always' do + Gitlab::Redis::Queues.with do |redis| + expect(redis).to receive(:sadd).with(queue_key, new_items) + expect(redis).to receive(:expire).with(queue_key, (lock_timeout + described_class::EXTRA_QUEUE_EXPIRE_WINDOW).to_i) + end + + subject + end + + it 'yields the new items with exclusive lease' do + uuid = 'test' + expect_to_obtain_exclusive_lease(lock_key, uuid, timeout: lock_timeout) + expect_to_cancel_exclusive_lease(lock_key, uuid) + + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args(match_array(new_items)) + end + + it 'returns the result and no items in the queue' do + expect(subject[:status]).to eq(:finished) + expect(subject[:new_items]).to be_empty + + Gitlab::Redis::Queues.with do |redis| + expect(redis.llen(queue_key)).to be(0) + end + end + + context 'when new items are enqueued during the process' do + it 'returns the result with newly added items' do + result = queue.safe_execute(new_items) do + queue.safe_execute(['C']) + end + + expect(result[:status]).to eq(:finished) + expect(result[:new_items]).to eq(['C']) + + Gitlab::Redis::Queues.with do |redis| + expect(redis.scard(queue_key)).to be(1) + end + end + end + + context 'when interger items are enqueued' do + let(:new_items) { [1, 2, 3] } + + it 'yields as String values' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args(%w[1 2 3]) + end + end + + context 'when the queue key does not exist in Redis' do + before do + allow(queue).to receive(:enqueue) { } + end + + it 'yields empty array' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args([]) + end + end + + context 'when the other process has already been working on the queue' do + before do + stub_exclusive_lease_taken(lock_key, timeout: lock_timeout) + end + + it 'does not yield the block' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .not_to yield_control + end + + it 'returns the result' do + expect(subject[:status]).to eq(:enqueued) + end + end + + context 'when a duplicate item is enqueued' do + it 'returns the poped items to the queue and raise an error' do + expect { |b| queue.safe_execute(%w[1 1 2 2], &b) } + .to yield_with_args(match_array(%w[1 2])) + end + end + + context 'when there are two queues' do + it 'enqueues items to each queue' do + queue_1 = described_class.new(namespace, '1') + queue_2 = described_class.new(namespace, '2') + + result_2 = nil + + result_1 = queue_1.safe_execute(['A']) do |_| + result_2 = queue_2.safe_execute(['B']) do |_| + queue_1.safe_execute(['C']) + queue_2.safe_execute(['D']) + end + end + + expect(result_1[:status]).to eq(:finished) + expect(result_1[:new_items]).to eq(['C']) + expect(result_2[:status]).to eq(:finished) + expect(result_2[:new_items]).to eq(['D']) + end + end + end +end diff --git a/spec/lib/gitlab/current_settings_spec.rb b/spec/lib/gitlab/current_settings_spec.rb index 909dbffa38f..db31e5280a3 100644 --- a/spec/lib/gitlab/current_settings_spec.rb +++ b/spec/lib/gitlab/current_settings_spec.rb @@ -80,7 +80,7 @@ describe Gitlab::CurrentSettings do # during the initialization phase of the test suite, so instead let's mock the internals of it expect(ActiveRecord::Base.connection).not_to receive(:active?) expect(ActiveRecord::Base.connection).not_to receive(:cached_table_exists?) - expect(ActiveRecord::Migrator).not_to receive(:needs_migration?) + expect_any_instance_of(ActiveRecord::MigrationContext).not_to receive(:needs_migration?) expect(ActiveRecord::QueryRecorder.new { described_class.current_application_settings }.count).to eq(0) end end @@ -109,7 +109,7 @@ describe Gitlab::CurrentSettings do context 'with pending migrations' do before do - expect(ActiveRecord::Migrator).to receive(:needs_migration?).and_return(true) + expect_any_instance_of(ActiveRecord::MigrationContext).to receive(:needs_migration?).and_return(true) end shared_examples 'a non-persisted ApplicationSetting object' do diff --git a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb index 48139c2f9dc..7833b9f387d 100644 --- a/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/create_issue_handler_spec.rb @@ -105,6 +105,7 @@ describe Gitlab::Email::Handler::CreateIssueHandler do context "when the issue could not be saved" do before do allow_any_instance_of(Issue).to receive(:persisted?).and_return(false) + allow_any_instance_of(Issue).to receive(:ensure_metrics).and_return(nil) end it "raises an InvalidIssueError" do diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index 978e64c4407..97ebb5f1554 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -176,6 +176,9 @@ describe Gitlab::Kubernetes::KubeClient do let(:rbac_client) { client.rbac_client } [ + :create_role, + :get_role, + :update_role, :create_cluster_role_binding, :get_cluster_role_binding, :update_cluster_role_binding diff --git a/spec/lib/gitlab/kubernetes/role_binding_spec.rb b/spec/lib/gitlab/kubernetes/role_binding_spec.rb index 50acee254cb..4c200eb545f 100644 --- a/spec/lib/gitlab/kubernetes/role_binding_spec.rb +++ b/spec/lib/gitlab/kubernetes/role_binding_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' describe Gitlab::Kubernetes::RoleBinding, '#generate' do let(:role_name) { 'edit' } + let(:role_kind) { 'ClusterRole' } let(:namespace) { 'my-namespace' } let(:service_account_name) { 'my-service-account' } @@ -20,7 +21,7 @@ describe Gitlab::Kubernetes::RoleBinding, '#generate' do let(:role_ref) do { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end @@ -37,6 +38,7 @@ describe Gitlab::Kubernetes::RoleBinding, '#generate' do described_class.new( name: "gitlab-#{namespace}", role_name: role_name, + role_kind: role_kind, namespace: namespace, service_account_name: service_account_name ).generate diff --git a/spec/lib/gitlab/kubernetes/role_spec.rb b/spec/lib/gitlab/kubernetes/role_spec.rb new file mode 100644 index 00000000000..3a5cd3b6704 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/role_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Kubernetes::Role do + let(:role) { described_class.new(name: name, namespace: namespace, rules: rules) } + let(:name) { 'example-name' } + let(:namespace) { 'example-namespace' } + + let(:rules) do + [{ + apiGroups: %w(hello.world), + resources: %w(oil diamonds coffee), + verbs: %w(say do walk run) + }] + end + + describe '#generate' do + subject { role.generate } + + let(:resource) do + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + it { is_expected.to eq(resource) } + end +end diff --git a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb index 18052b1991c..c5fc74afea5 100644 --- a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb +++ b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb @@ -9,12 +9,13 @@ describe Gitlab::MarkdownCache::ActiveRecord::Extension do cache_markdown_field :title, whitelisted: true cache_markdown_field :description, pipeline: :single_line - attr_accessor :author, :project + attribute :author + attribute :project end end let(:cache_version) { Gitlab::MarkdownCache::CACHE_COMMONMARK_VERSION << 16 } - let(:thing) { klass.new(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) { klass.create(title: markdown, title_html: html, cached_markdown_version: cache_version) } let(:markdown) { '`Foo`' } let(:html) { '<p data-sourcepos="1:1-1:5" dir="auto"><code>Foo</code></p>' } @@ -37,7 +38,7 @@ describe Gitlab::MarkdownCache::ActiveRecord::Extension do end context 'a changed markdown field' do - let(:thing) { klass.new(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) { klass.create(title: markdown, title_html: html, cached_markdown_version: cache_version) } before do thing.title = updated_markdown diff --git a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb b/spec/lib/gitlab/phabricator_import/cache/map_spec.rb index 52c7a02219f..b6629fad453 100644 --- a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb +++ b/spec/lib/gitlab/phabricator_import/cache/map_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Gitlab::PhabricatorImport::Cache::Map, :clean_gitlab_redis_cache do @@ -28,6 +30,21 @@ describe Gitlab::PhabricatorImport::Cache::Map, :clean_gitlab_redis_cache do expect(ttl).to be > 10.seconds end + + it 'sets the object in redis once if a block was given and nothing was cached' do + issue = create(:issue, project: project) + + expect(map.get_gitlab_model('does not exist') { issue }).to eq(issue) + + expect { |b| map.get_gitlab_model('does not exist', &b) } + .not_to yield_control + end + + it 'does not cache `nil` objects' do + expect(map).not_to receive(:set_gitlab_model) + + map.get_gitlab_model('does not exist') { nil } + end end describe '#set_gitlab_model' do diff --git a/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb new file mode 100644 index 00000000000..e88eec2c393 --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Conduit::User do + let(:user_client) do + described_class.new(phabricator_url: 'https://see-ya-later.phabricator', api_token: 'api-token') + end + + describe '#users' do + let(:fake_client) { double('Phabricator client') } + + before do + allow(user_client).to receive(:client).and_return(fake_client) + end + + it 'calls the api with the correct params' do + expected_params = { + constraints: { phids: ['phid-1', 'phid-2'] } + } + + expect(fake_client).to receive(:get).with('user.search', + params: expected_params) + + user_client.users(['phid-1', 'phid-2']) + end + + it 'returns an array of parsed responses' do + response = Gitlab::PhabricatorImport::Conduit::Response + .new(fixture_file('phabricator_responses/user.search.json')) + + allow(fake_client).to receive(:get).and_return(response) + + expect(user_client.users(%w[some phids])).to match_array([an_instance_of(Gitlab::PhabricatorImport::Conduit::UsersResponse)]) + end + + it 'performs multiple requests if more phids than the maximum page size are passed' do + stub_const('Gitlab::PhabricatorImport::Conduit::User::MAX_PAGE_SIZE', 1) + first_params = { constraints: { phids: ['phid-1'] } } + second_params = { constraints: { phids: ['phid-2'] } } + + expect(fake_client).to receive(:get).with('user.search', + params: first_params).once + expect(fake_client).to receive(:get).with('user.search', + params: second_params).once + + user_client.users(['phid-1', 'phid-2']) + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb new file mode 100644 index 00000000000..00778ad90fd --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Conduit::UsersResponse do + let(:conduit_response) do + Gitlab::PhabricatorImport::Conduit::Response + .new(JSON.parse(fixture_file('phabricator_responses/user.search.json'))) + end + + subject(:response) { described_class.new(conduit_response) } + + describe '#users' do + it 'builds the correct users representation' do + tasks = response.users + + usernames = tasks.map(&:username) + + expect(usernames).to contain_exactly('jane', 'john') + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb index 2412cf76f79..667321409da 100644 --- a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb +++ b/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Gitlab::PhabricatorImport::Issues::Importer do - set(:project) { create(:project) } + let(:project) { create(:project) } let(:response) do Gitlab::PhabricatorImport::Conduit::TasksResponse.new( @@ -15,7 +15,6 @@ describe Gitlab::PhabricatorImport::Issues::Importer do before do client = instance_double(Gitlab::PhabricatorImport::Conduit::Maniphest) - allow(client).to receive(:tasks).and_return(response) allow(importer).to receive(:client).and_return(client) end @@ -34,20 +33,29 @@ describe Gitlab::PhabricatorImport::Issues::Importer do importer.execute end - it 'schedules the next batch if there is one' do - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .to receive(:schedule).with(project.id, response.pagination.next_page) + context 'stubbed task import' do + before do + # Stub out the actual importing so we don't perform aditional requests + expect_next_instance_of(Gitlab::PhabricatorImport::Issues::TaskImporter) do |task_importer| + allow(task_importer).to receive(:execute) + end.at_least(1) + end - importer.execute - end + it 'schedules the next batch if there is one' do + expect(Gitlab::PhabricatorImport::ImportTasksWorker) + .to receive(:schedule).with(project.id, response.pagination.next_page) - it 'does not reschedule when there is no next page' do - allow(response.pagination).to receive(:has_next_page?).and_return(false) + importer.execute + end - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .not_to receive(:schedule) + it 'does not reschedule when there is no next page' do + allow(response.pagination).to receive(:has_next_page?).and_return(false) - importer.execute + expect(Gitlab::PhabricatorImport::ImportTasksWorker) + .not_to receive(:schedule) + + importer.execute + end end end end diff --git a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb index 1625604e754..06ed264e781 100644 --- a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb +++ b/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb @@ -12,6 +12,8 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do 'description' => { 'raw' => '# This is markdown\n it can contain more text.' }, + 'authorPHID' => 'PHID-USER-456', + 'ownerPHID' => 'PHID-USER-123', 'dateCreated' => '1518688921', 'dateClosed' => '1518789995' } @@ -19,9 +21,18 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do ) end + subject(:importer) { described_class.new(project, task) } + describe '#execute' do + let(:fake_user_finder) { instance_double(Gitlab::PhabricatorImport::UserFinder) } + + before do + allow(fake_user_finder).to receive(:find) + allow(importer).to receive(:user_finder).and_return(fake_user_finder) + end + it 'creates the issue with the expected attributes' do - issue = described_class.new(project, task).execute + issue = importer.execute expect(issue.project).to eq(project) expect(issue).to be_persisted @@ -34,21 +45,38 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do end it 'does not recreate the issue when called multiple times' do - expect { described_class.new(project, task).execute } + expect { importer.execute } .to change { project.issues.reload.size }.from(0).to(1) - expect { described_class.new(project, task).execute } + expect { importer.execute } .not_to change { project.issues.reload.size } end it 'does not trigger a save when the object did not change' do existing_issue = create(:issue, task.issue_attributes.merge(author: User.ghost)) - importer = described_class.new(project, task) allow(importer).to receive(:issue).and_return(existing_issue) expect(existing_issue).not_to receive(:save!) importer.execute end + + it 'links the author if the author can be found' do + author = create(:user) + expect(fake_user_finder).to receive(:find).with('PHID-USER-456').and_return(author) + + issue = importer.execute + + expect(issue.author).to eq(author) + end + + it 'links an assignee if the user can be found' do + assignee = create(:user) + expect(fake_user_finder).to receive(:find).with('PHID-USER-123').and_return(assignee) + + issue = importer.execute + + expect(issue.assignees).to include(assignee) + end end end diff --git a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb b/spec/lib/gitlab/phabricator_import/representation/task_spec.rb index dfbd8c546eb..5603a6961d6 100644 --- a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb +++ b/spec/lib/gitlab/phabricator_import/representation/task_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Gitlab::PhabricatorImport::Representation::Task do @@ -7,6 +9,8 @@ describe Gitlab::PhabricatorImport::Representation::Task do 'phid' => 'the-phid', 'fields' => { 'name' => 'Title'.ljust(257, '.'), # A string padded to 257 chars + 'authorPHID' => 'a phid', + 'ownerPHID' => 'another user phid', 'description' => { 'raw' => '# This is markdown\n it can contain more text.' }, @@ -30,4 +34,16 @@ describe Gitlab::PhabricatorImport::Representation::Task do expect(task.issue_attributes).to eq(expected_attributes) end end + + describe '#author_phid' do + it 'returns the correct field' do + expect(task.author_phid).to eq('a phid') + end + end + + describe '#owner_phid' do + it 'returns the correct field' do + expect(task.owner_phid).to eq('another user phid') + end + end end diff --git a/spec/lib/gitlab/phabricator_import/representation/user_spec.rb b/spec/lib/gitlab/phabricator_import/representation/user_spec.rb new file mode 100644 index 00000000000..f52467a0cf1 --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/representation/user_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Representation::User do + subject(:user) do + described_class.new( + { + 'phid' => 'the-phid', + 'fields' => { + 'username' => 'the-username' + } + } + ) + end + + describe '#phabricator_id' do + it 'returns the phabricator id' do + expect(user.phabricator_id).to eq('the-phid') + end + end + + describe '#username' do + it 'returns the username' do + expect(user.username).to eq('the-username') + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb new file mode 100644 index 00000000000..096321cda5f --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe Gitlab::PhabricatorImport::UserFinder, :clean_gitlab_redis_cache do + let(:project) { create(:project, namespace: create(:group)) } + subject(:finder) { described_class.new(project, ['first-phid', 'second-phid']) } + + before do + project.namespace.add_developer(existing_user) + end + + describe '#find' do + let!(:existing_user) { create(:user, username: 'existing-user') } + let(:cache) { Gitlab::PhabricatorImport::Cache::Map.new(project) } + + before do + allow(finder).to receive(:object_map).and_return(cache) + end + + context 'for a cached phid' do + before do + cache.set_gitlab_model(existing_user, 'first-phid') + end + + it 'returns the existing user' do + expect(finder.find('first-phid')).to eq(existing_user) + end + + it 'does not perform a find using the API' do + expect(finder).not_to receive(:find_user_for_phid) + + finder.find('first-phid') + end + + it 'excludes the phid from the request if one needs to be made' do + client = instance_double(Gitlab::PhabricatorImport::Conduit::User) + allow(finder).to receive(:client).and_return(client) + + expect(client).to receive(:users).with(['second-phid']).and_return([]) + + finder.find('first-phid') + finder.find('second-phid') + end + end + + context 'when the phid is not cached' do + let(:response) do + [ + instance_double( + Gitlab::PhabricatorImport::Conduit::UsersResponse, + users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'second-phid', username: 'existing-user')] + ), + instance_double( + Gitlab::PhabricatorImport::Conduit::UsersResponse, + users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'first-phid', username: 'other-user')] + ) + ] + end + let(:client) do + client = instance_double(Gitlab::PhabricatorImport::Conduit::User) + allow(client).to receive(:users).and_return(response) + + client + end + + before do + allow(finder).to receive(:client).and_return(client) + end + + it 'loads the users from the API once' do + expect(client).to receive(:users).and_return(response).once + + expect(finder.find('second-phid')).to eq(existing_user) + expect(finder.find('first-phid')).to be_nil + end + + it 'adds found users to the cache' do + expect { finder.find('second-phid') } + .to change { cache.get_gitlab_model('second-phid') } + .from(nil).to(existing_user) + end + + it 'only returns users that are members of the project' do + create(:user, username: 'other-user') + + expect(finder.find('first-phid')).to eq(nil) + end + end + end +end diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index 253366e0789..f8b0cbfb6f6 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -353,7 +353,7 @@ describe Gitlab::UrlBlocker do end end - describe '#validate_hostname!' do + describe '#validate_hostname' do let(:ip_addresses) do [ '2001:db8:1f70::999:de8:7648:6e8', @@ -378,7 +378,7 @@ describe Gitlab::UrlBlocker do it 'does not raise error for valid Ip addresses' do ip_addresses.each do |ip| - expect { described_class.send(:validate_hostname!, ip) }.not_to raise_error + expect { described_class.send(:validate_hostname, ip) }.not_to raise_error end end end diff --git a/spec/migrations/active_record/schema_spec.rb b/spec/migrations/active_record/schema_spec.rb index fbf5d387d0e..bc246f88685 100644 --- a/spec/migrations/active_record/schema_spec.rb +++ b/spec/migrations/active_record/schema_spec.rb @@ -15,7 +15,7 @@ describe ActiveRecord::Schema do it '> schema version equals last migration timestamp' do defined_schema_version = File.open(Rails.root.join('db', 'schema.rb')) do |file| file.find { |line| line =~ /ActiveRecord::Schema.define/ } - end.match(/(\d+)/)[0].to_i + end.match(/(\d{4}_\d{2}_\d{2}_\d{6})/)[0].to_i expect(defined_schema_version).to eq(latest_migration_timestamp) end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index d98db024f73..78862de0657 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2013,6 +2013,7 @@ describe Ci::Build do { key: 'CI', value: 'true', public: true, masked: false }, { key: 'GITLAB_CI', value: 'true', public: true, masked: false }, { key: 'GITLAB_FEATURES', value: project.licensed_features.join(','), public: true, masked: false }, + { key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host, public: true, masked: false }, { key: 'CI_SERVER_NAME', value: 'GitLab', public: true, masked: false }, { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true, masked: false }, { key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s, public: true, masked: false }, diff --git a/spec/models/commit_range_spec.rb b/spec/models/commit_range_spec.rb index b96ca89c893..4a524b585e1 100644 --- a/spec/models/commit_range_spec.rb +++ b/spec/models/commit_range_spec.rb @@ -139,8 +139,8 @@ describe CommitRange do end describe '#has_been_reverted?' do - let(:issue) { create(:issue) } - let(:user) { issue.author } + let(:user) { create(:user) } + let(:issue) { create(:issue, author: user, project: project) } it 'returns true if the commit has been reverted' do create(:note_on_issue, @@ -149,9 +149,11 @@ describe CommitRange do note: commit1.revert_description(user), project: issue.project) - expect_any_instance_of(Commit).to receive(:reverts_commit?) - .with(commit1, user) - .and_return(true) + expect_next_instance_of(Commit) do |commit| + expect(commit).to receive(:reverts_commit?) + .with(commit1, user) + .and_return(true) + end expect(commit1.has_been_reverted?(user, issue.notes_with_associations)).to eq(true) end diff --git a/spec/models/concerns/cache_markdown_field_spec.rb b/spec/models/concerns/cache_markdown_field_spec.rb index 0e5fb2b5153..9a12c3d6965 100644 --- a/spec/models/concerns/cache_markdown_field_spec.rb +++ b/spec/models/concerns/cache_markdown_field_spec.rb @@ -198,6 +198,36 @@ describe CacheMarkdownField, :clean_gitlab_redis_cache do end end end + + describe '#updated_cached_html_for' do + let(:thing) { klass.new(description: markdown, description_html: html, cached_markdown_version: cache_version) } + + context 'when the markdown cache is outdated' do + before do + thing.cached_markdown_version += 1 + end + + it 'calls #refresh_markdown_cache' do + expect(thing).to receive(:refresh_markdown_cache) + + expect(thing.updated_cached_html_for(:description)).to eq(html) + end + end + + context 'when the markdown field does not exist' do + it 'returns nil' do + expect(thing.updated_cached_html_for(:something)).to eq(nil) + end + end + + context 'when the markdown cache is up to date' do + it 'does not call #refresh_markdown_cache' do + expect(thing).not_to receive(:refresh_markdown_cache) + + expect(thing.updated_cached_html_for(:description)).to eq(html) + end + end + end end context 'for Active record classes' do diff --git a/spec/models/concerns/routable_spec.rb b/spec/models/concerns/routable_spec.rb index 1fb0dd5030c..31163a5bb5c 100644 --- a/spec/models/concerns/routable_spec.rb +++ b/spec/models/concerns/routable_spec.rb @@ -15,23 +15,46 @@ describe Group, 'Routable' do end describe 'Callbacks' do - it 'creates route record on create' do - expect(group.route.path).to eq(group.path) - expect(group.route.name).to eq(group.name) - end + context 'for a group' do + it 'creates route record on create' do + expect(group.route.path).to eq(group.path) + expect(group.route.name).to eq(group.name) + end + + it 'updates route record on path change' do + group.update(path: 'wow', name: 'much') - it 'updates route record on path change' do - group.update(path: 'wow', name: 'much') + expect(group.route.path).to eq('wow') + expect(group.route.name).to eq('much') + end + + it 'ensure route path uniqueness across different objects' do + create(:group, parent: group, path: 'xyz') + duplicate = build(:project, namespace: group, path: 'xyz') - expect(group.route.path).to eq('wow') - expect(group.route.name).to eq('much') + expect { duplicate.save! }.to raise_error(ActiveRecord::RecordInvalid, 'Validation failed: Path has already been taken') + end end - it 'ensure route path uniqueness across different objects' do - create(:group, parent: group, path: 'xyz') - duplicate = build(:project, namespace: group, path: 'xyz') + context 'for a user' do + let(:user) { create(:user, username: 'jane', name: "Jane Doe") } - expect { duplicate.save! }.to raise_error(ActiveRecord::RecordInvalid, 'Validation failed: Path has already been taken') + it 'creates the route for a record on create' do + expect(user.namespace.name).to eq('Jane Doe') + expect(user.namespace.path).to eq('jane') + end + + it 'updates routes and nested routes on name change' do + project = create(:project, path: 'work-stuff', name: 'Work stuff', namespace: user.namespace) + + user.update!(username: 'jaen', name: 'Jaen Did') + project.reload + + expect(user.namespace.name).to eq('Jaen Did') + expect(user.namespace.path).to eq('jaen') + expect(project.full_name).to eq('Jaen Did / Work stuff') + expect(project.full_path).to eq('jaen/work-stuff') + end end end diff --git a/spec/models/cycle_analytics/issue_spec.rb b/spec/models/cycle_analytics/issue_spec.rb index 7b18e24a351..4ccbdf29df6 100644 --- a/spec/models/cycle_analytics/issue_spec.rb +++ b/spec/models/cycle_analytics/issue_spec.rb @@ -24,7 +24,7 @@ describe 'CycleAnalytics#issue' do ["list label added to issue", -> (context, data) do if data[:issue].persisted? - data[:issue].update(label_ids: [context.create(:label, lists: [context.create(:list)]).id]) + data[:issue].update(label_ids: [context.create(:list).label_id]) end end]], post_fn: -> (context, data) do diff --git a/spec/models/cycle_analytics/plan_spec.rb b/spec/models/cycle_analytics/plan_spec.rb index 73fc98ba6cf..c99c38e9cf3 100644 --- a/spec/models/cycle_analytics/plan_spec.rb +++ b/spec/models/cycle_analytics/plan_spec.rb @@ -25,7 +25,7 @@ describe 'CycleAnalytics#plan' do end], ["list label added to issue", -> (context, data) do - data[:issue].update(label_ids: [context.create(:label, lists: [context.create(:list)]).id]) + data[:issue].update(label_ids: [context.create(:list).label_id]) end]], end_time_conditions: [["issue mentioned in a commit", -> (context, data) do diff --git a/spec/models/issue/metrics_spec.rb b/spec/models/issue/metrics_spec.rb index 07858fe8a70..7aa0d97b194 100644 --- a/spec/models/issue/metrics_spec.rb +++ b/spec/models/issue/metrics_spec.rb @@ -32,7 +32,7 @@ describe Issue::Metrics do context "list labels" do it "records the first time an issue is associated with a list label" do - list_label = create(:label, lists: [create(:list)]) + list_label = create(:list).label time = Time.now Timecop.freeze(time) { subject.update(label_ids: [list_label.id]) } metrics = subject.metrics @@ -43,9 +43,9 @@ describe Issue::Metrics do it "does not record the second time an issue is associated with a list label" do time = Time.now - first_list_label = create(:label, lists: [create(:list)]) + first_list_label = create(:list).label Timecop.freeze(time) { subject.update(label_ids: [first_list_label.id]) } - second_list_label = create(:label, lists: [create(:list)]) + second_list_label = create(:list).label Timecop.freeze(time + 5.hours) { subject.update(label_ids: [second_list_label.id]) } metrics = subject.metrics diff --git a/spec/models/note_spec.rb b/spec/models/note_spec.rb index 7a1ab20186a..03003e3dd7d 100644 --- a/spec/models/note_spec.rb +++ b/spec/models/note_spec.rb @@ -177,6 +177,7 @@ describe Note do pipeline: :note, cache_key: [note1, "note"], project: note1.project, + rendered: note1.note_html, author: note1.author } }]).and_call_original @@ -189,6 +190,7 @@ describe Note do pipeline: :note, cache_key: [note2, "note"], project: note2.project, + rendered: note2.note_html, author: note2.author } }]).and_call_original diff --git a/spec/models/project_services/bugzilla_service_spec.rb b/spec/models/project_services/bugzilla_service_spec.rb index d5b0f94f461..74c85a13c88 100644 --- a/spec/models/project_services/bugzilla_service_spec.rb +++ b/spec/models/project_services/bugzilla_service_spec.rb @@ -44,7 +44,9 @@ describe BugzillaService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:bugzilla_service, properties: properties) } + let(:service) do + create(:bugzilla_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -60,7 +62,7 @@ describe BugzillaService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:bugzilla_service, title: title, description: description, properties: properties) + create(:bugzilla_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/custom_issue_tracker_service_spec.rb b/spec/models/project_services/custom_issue_tracker_service_spec.rb index 56b0bda6626..5259357a254 100644 --- a/spec/models/project_services/custom_issue_tracker_service_spec.rb +++ b/spec/models/project_services/custom_issue_tracker_service_spec.rb @@ -58,7 +58,9 @@ describe CustomIssueTrackerService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:custom_issue_tracker_service, properties: properties) } + let(:service) do + create(:custom_issue_tracker_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -74,7 +76,7 @@ describe CustomIssueTrackerService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:custom_issue_tracker_service, title: title, description: description, properties: properties) + create(:custom_issue_tracker_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/gitlab_issue_tracker_service_spec.rb b/spec/models/project_services/gitlab_issue_tracker_service_spec.rb index a3726f09dc5..0c4fc290a13 100644 --- a/spec/models/project_services/gitlab_issue_tracker_service_spec.rb +++ b/spec/models/project_services/gitlab_issue_tracker_service_spec.rb @@ -61,7 +61,9 @@ describe GitlabIssueTrackerService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:gitlab_issue_tracker_service, properties: properties) } + let(:service) do + create(:gitlab_issue_tracker_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -77,7 +79,7 @@ describe GitlabIssueTrackerService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:gitlab_issue_tracker_service, title: title, description: description, properties: properties) + create(:gitlab_issue_tracker_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/jira_service_spec.rb b/spec/models/project_services/jira_service_spec.rb index 9b122d85293..235cf314af5 100644 --- a/spec/models/project_services/jira_service_spec.rb +++ b/spec/models/project_services/jira_service_spec.rb @@ -145,7 +145,9 @@ describe JiraService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:jira_service, properties: properties) } + let(:service) do + create(:jira_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -161,7 +163,7 @@ describe JiraService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:jira_service, title: title, description: description, properties: properties) + create(:jira_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/redmine_service_spec.rb b/spec/models/project_services/redmine_service_spec.rb index 806e3695962..c1ee6546b12 100644 --- a/spec/models/project_services/redmine_service_spec.rb +++ b/spec/models/project_services/redmine_service_spec.rb @@ -50,7 +50,9 @@ describe RedmineService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:redmine_service, properties: properties) } + let(:service) do + create(:redmine_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -66,7 +68,7 @@ describe RedmineService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:redmine_service, title: title, description: description, properties: properties) + create(:redmine_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/youtrack_service_spec.rb b/spec/models/project_services/youtrack_service_spec.rb index b47ef6702b4..c48bf487af0 100644 --- a/spec/models/project_services/youtrack_service_spec.rb +++ b/spec/models/project_services/youtrack_service_spec.rb @@ -47,7 +47,9 @@ describe YoutrackService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:youtrack_service, properties: properties) } + let(:service) do + create(:youtrack_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -63,7 +65,7 @@ describe YoutrackService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:youtrack_service, title: title, description: description, properties: properties) + create(:youtrack_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 3d967aa4ab8..12dff440ce2 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -2426,16 +2426,15 @@ describe Repository do # Gets the commit oid, and warms the cache oid = project.commit.id - expect(Gitlab::Git::Commit).not_to receive(:find).once + expect(Gitlab::Git::Commit).to receive(:find).once - project.commit_by(oid: oid) + 2.times { project.commit_by(oid: oid) } end it 'caches nil values' do expect(Gitlab::Git::Commit).to receive(:find).once - project.commit_by(oid: '1' * 40) - project.commit_by(oid: '1' * 40) + 2.times { project.commit_by(oid: '1' * 40) } end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a4d177da0be..e4f84172215 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -2931,7 +2931,7 @@ describe User do let(:user) { create(:user, username: username) } context 'when the user is updated' do - context 'when the username is changed' do + context 'when the username or name is changed' do let(:new_username) { 'bar' } it 'changes the namespace (just to compare to when username is not changed)' do @@ -2942,16 +2942,24 @@ describe User do end.to change { user.namespace.updated_at } end - it 'updates the namespace name' do + it 'updates the namespace path when the username was changed' do user.update!(username: new_username) - expect(user.namespace.name).to eq(new_username) + expect(user.namespace.path).to eq(new_username) end - it 'updates the namespace path' do - user.update!(username: new_username) + it 'updates the namespace name if the name was changed' do + user.update!(name: 'New name') - expect(user.namespace.path).to eq(new_username) + expect(user.namespace.name).to eq('New name') + end + + it 'updates nested routes for the namespace if the name was changed' do + project = create(:project, namespace: user.namespace) + + user.update!(name: 'New name') + + expect(project.route.reload.name).to include('New name') end context 'when there is a validation error (namespace name taken) while updating namespace' do diff --git a/spec/requests/api/group_clusters_spec.rb b/spec/requests/api/group_clusters_spec.rb new file mode 100644 index 00000000000..46e3dd650cc --- /dev/null +++ b/spec/requests/api/group_clusters_spec.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::GroupClusters do + include KubernetesHelpers + + let(:current_user) { create(:user) } + let(:developer_user) { create(:user) } + let(:group) { create(:group, :private) } + + before do + group.add_developer(developer_user) + group.add_maintainer(current_user) + end + + describe 'GET /groups/:id/clusters' do + let!(:extra_cluster) { create(:cluster, :provided_by_gcp, :group) } + + let!(:clusters) do + create_list(:cluster, 5, :provided_by_gcp, :group, :production_environment, + groups: [group]) + end + + context 'non-authorized user' do + it 'responds with 403' do + get api("/groups/#{group.id}/clusters", developer_user) + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'authorized user' do + before do + get api("/groups/#{group.id}/clusters", current_user) + end + + it 'responds with 200' do + expect(response).to have_gitlab_http_status(200) + end + + it 'includes pagination headers' do + expect(response).to include_pagination_headers + end + + it 'only include authorized clusters' do + cluster_ids = json_response.map { |cluster| cluster['id'] } + + expect(cluster_ids).to match_array(clusters.pluck(:id)) + expect(cluster_ids).not_to include(extra_cluster.id) + end + end + end + + describe 'GET /groups/:id/clusters/:cluster_id' do + let(:cluster_id) { cluster.id } + + let(:platform_kubernetes) do + create(:cluster_platform_kubernetes, :configured) + end + + let(:cluster) do + create(:cluster, :group, :provided_by_gcp, :with_domain, + platform_kubernetes: platform_kubernetes, + user: current_user, + groups: [group]) + end + + context 'non-authorized user' do + it 'responds with 403' do + get api("/groups/#{group.id}/clusters/#{cluster_id}", developer_user) + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'authorized user' do + before do + get api("/groups/#{group.id}/clusters/#{cluster_id}", current_user) + end + + it 'returns specific cluster' do + expect(json_response['id']).to eq(cluster.id) + end + + it 'returns cluster information' do + expect(json_response['provider_type']).to eq('gcp') + expect(json_response['platform_type']).to eq('kubernetes') + expect(json_response['environment_scope']).to eq('*') + expect(json_response['cluster_type']).to eq('group_type') + expect(json_response['domain']).to eq('example.com') + end + + it 'returns group information' do + cluster_group = json_response['group'] + + expect(cluster_group['id']).to eq(group.id) + expect(cluster_group['name']).to eq(group.name) + expect(cluster_group['web_url']).to eq(group.web_url) + end + + it 'returns kubernetes platform information' do + platform = json_response['platform_kubernetes'] + + expect(platform['api_url']).to eq('https://kubernetes.example.com') + expect(platform['ca_cert']).to be_present + end + + it 'returns user information' do + user = json_response['user'] + + expect(user['id']).to eq(current_user.id) + expect(user['username']).to eq(current_user.username) + end + + it 'returns GCP provider information' do + gcp_provider = json_response['provider_gcp'] + + expect(gcp_provider['cluster_id']).to eq(cluster.id) + expect(gcp_provider['status_name']).to eq('created') + expect(gcp_provider['gcp_project_id']).to eq('test-gcp-project') + expect(gcp_provider['zone']).to eq('us-central1-a') + expect(gcp_provider['machine_type']).to eq('n1-standard-2') + expect(gcp_provider['num_nodes']).to eq(3) + expect(gcp_provider['endpoint']).to eq('111.111.111.111') + end + + context 'when cluster has no provider' do + let(:cluster) do + create(:cluster, :group, :provided_by_user, + groups: [group]) + end + + it 'does not include GCP provider info' do + expect(json_response['provider_gcp']).not_to be_present + end + end + + context 'with non-existing cluster' do + let(:cluster_id) { 123 } + + it 'returns 404' do + expect(response).to have_gitlab_http_status(404) + end + end + end + end + + shared_context 'kubernetes calls stubbed' do + before do + stub_kubeclient_discover(api_url) + end + end + + describe 'POST /groups/:id/clusters/user' do + include_context 'kubernetes calls stubbed' + + let(:api_url) { 'https://kubernetes.example.com' } + let(:authorization_type) { 'rbac' } + + let(:platform_kubernetes_attributes) do + { + api_url: api_url, + token: 'sample-token', + authorization_type: authorization_type + } + end + + let(:cluster_params) do + { + name: 'test-cluster', + domain: 'domain.example.com', + managed: false, + platform_kubernetes_attributes: platform_kubernetes_attributes + } + end + + context 'non-authorized user' do + it 'responds with 403' do + post api("/groups/#{group.id}/clusters/user", developer_user), params: cluster_params + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'authorized user' do + before do + post api("/groups/#{group.id}/clusters/user", current_user), params: cluster_params + end + + context 'with valid params' do + it 'responds with 201' do + expect(response).to have_gitlab_http_status(201) + end + + it 'creates a new Cluster::Cluster' do + cluster_result = Clusters::Cluster.find(json_response["id"]) + platform_kubernetes = cluster_result.platform + + expect(cluster_result).to be_user + expect(cluster_result).to be_kubernetes + expect(cluster_result.group).to eq(group) + expect(cluster_result.name).to eq('test-cluster') + expect(cluster_result.domain).to eq('domain.example.com') + expect(cluster_result.managed).to be_falsy + expect(platform_kubernetes.rbac?).to be_truthy + expect(platform_kubernetes.api_url).to eq(api_url) + expect(platform_kubernetes.token).to eq('sample-token') + end + end + + context 'when user does not indicate authorization type' do + let(:platform_kubernetes_attributes) do + { + api_url: api_url, + token: 'sample-token' + } + end + + it 'defaults to RBAC' do + cluster_result = Clusters::Cluster.find(json_response['id']) + + expect(cluster_result.platform_kubernetes.rbac?).to be_truthy + end + end + + context 'when user sets authorization type as ABAC' do + let(:authorization_type) { 'abac' } + + it 'creates an ABAC cluster' do + cluster_result = Clusters::Cluster.find(json_response['id']) + + expect(cluster_result.platform.abac?).to be_truthy + end + end + + context 'with invalid params' do + let(:api_url) { 'invalid_api_url' } + + it 'responds with 400' do + expect(response).to have_gitlab_http_status(400) + end + + it 'does not create a new Clusters::Cluster' do + expect(group.reload.clusters).to be_empty + end + + it 'returns validation errors' do + expect(json_response['message']['platform_kubernetes.api_url'].first).to be_present + end + end + end + + context 'when user tries to add multiple clusters' do + before do + create(:cluster, :provided_by_gcp, :group, + groups: [group]) + + post api("/groups/#{group.id}/clusters/user", current_user), params: cluster_params + end + + it 'responds with 400' do + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']['base'].first).to include('Instance does not support multiple Kubernetes clusters') + end + end + + context 'non-authorized user' do + before do + post api("/groups/#{group.id}/clusters/user", developer_user), params: cluster_params + end + + it 'responds with 403' do + expect(response).to have_gitlab_http_status(403) + + expect(json_response['message']).to eq('403 Forbidden') + end + end + end + + describe 'PUT /groups/:id/clusters/:cluster_id' do + include_context 'kubernetes calls stubbed' + + let(:api_url) { 'https://kubernetes.example.com' } + + let(:update_params) do + { + domain: domain, + platform_kubernetes_attributes: platform_kubernetes_attributes + } + end + + let(:domain) { 'new-domain.com' } + let(:platform_kubernetes_attributes) { {} } + + let(:cluster) do + create(:cluster, :group, :provided_by_gcp, + groups: [group], domain: 'old-domain.com') + end + + context 'non-authorized user' do + it 'responds with 403' do + put api("/groups/#{group.id}/clusters/#{cluster.id}", developer_user), params: update_params + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'authorized user' do + before do + put api("/groups/#{group.id}/clusters/#{cluster.id}", current_user), params: update_params + + cluster.reload + end + + context 'with valid params' do + it 'responds with 200' do + expect(response).to have_gitlab_http_status(200) + end + + it 'updates cluster attributes' do + expect(cluster.domain).to eq('new-domain.com') + end + end + + context 'with invalid params' do + let(:domain) { 'invalid domain' } + + it 'responds with 400' do + expect(response).to have_gitlab_http_status(400) + end + + it 'does not update cluster attributes' do + expect(cluster.domain).to eq('old-domain.com') + end + + it 'returns validation errors' do + expect(json_response['message']['domain'].first).to match('contains invalid characters (valid characters: [a-z0-9\\-])') + end + end + + context 'with a GCP cluster' do + context 'when user tries to change GCP specific fields' do + let(:platform_kubernetes_attributes) do + { + api_url: 'https://new-api-url.com', + token: 'new-sample-token' + } + end + + it 'responds with 400' do + expect(response).to have_gitlab_http_status(400) + end + + it 'returns validation error' do + expect(json_response['message']['platform_kubernetes.base'].first).to eq('Cannot modify managed Kubernetes cluster') + end + end + + context 'when user tries to change domain' do + let(:domain) { 'new-domain.com' } + + it 'responds with 200' do + expect(response).to have_gitlab_http_status(200) + end + end + end + + context 'with an user cluster' do + let(:api_url) { 'https://new-api-url.com' } + + let(:cluster) do + create(:cluster, :group, :provided_by_user, + groups: [group]) + end + + let(:platform_kubernetes_attributes) do + { + api_url: api_url, + token: 'new-sample-token' + } + end + + let(:update_params) do + { + name: 'new-name', + platform_kubernetes_attributes: platform_kubernetes_attributes + } + end + + it 'responds with 200' do + expect(response).to have_gitlab_http_status(200) + end + + it 'updates platform kubernetes attributes' do + platform_kubernetes = cluster.platform_kubernetes + + expect(cluster.name).to eq('new-name') + expect(platform_kubernetes.api_url).to eq('https://new-api-url.com') + expect(platform_kubernetes.token).to eq('new-sample-token') + end + end + + context 'with a cluster that does not belong to user' do + let(:cluster) { create(:cluster, :group, :provided_by_user) } + + it 'responds with 404' do + expect(response).to have_gitlab_http_status(404) + end + end + end + end + + describe 'DELETE /groups/:id/clusters/:cluster_id' do + let(:cluster_params) { { cluster_id: cluster.id } } + + let(:cluster) do + create(:cluster, :group, :provided_by_gcp, + groups: [group]) + end + + context 'non-authorized user' do + it 'responds with 403' do + delete api("/groups/#{group.id}/clusters/#{cluster.id}", developer_user), params: cluster_params + + expect(response).to have_gitlab_http_status(403) + end + end + + context 'authorized user' do + before do + delete api("/groups/#{group.id}/clusters/#{cluster.id}", current_user), params: cluster_params + end + + it 'responds with 204' do + expect(response).to have_gitlab_http_status(204) + end + + it 'deletes the cluster' do + expect(Clusters::Cluster.exists?(id: cluster.id)).to be_falsy + end + + context 'with a cluster that does not belong to user' do + let(:cluster) { create(:cluster, :group, :provided_by_user) } + + it 'responds with 404' do + expect(response).to have_gitlab_http_status(404) + end + end + end + end +end diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index fcbff19bd61..3ab1818bebb 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -474,7 +474,7 @@ describe API::Internal do 'ssh', { authentication_abilities: [:read_project, :download_code, :push_code], - namespace_path: project.namespace.name, + namespace_path: project.namespace.path, project_path: project.path, redirected_path: nil } @@ -753,7 +753,7 @@ describe API::Internal do end describe 'GET /internal/merge_request_urls' do - let(:repo_name) { "#{project.namespace.name}/#{project.path}" } + let(:repo_name) { "#{project.full_path}" } let(:changes) { URI.escape("#{Gitlab::Git::BLANK_SHA} 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/new_branch") } before do @@ -765,7 +765,7 @@ describe API::Internal do expect(json_response).to match [{ "branch_name" => "new_branch", - "url" => "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch", + "url" => "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch", "new_merge_request" => true }] end @@ -786,7 +786,7 @@ describe API::Internal do expect(json_response).to match [{ "branch_name" => "new_branch", - "url" => "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch", + "url" => "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch", "new_merge_request" => true }] end @@ -927,7 +927,7 @@ describe API::Internal do expect(json_response['merge_request_urls']).to match [{ "branch_name" => branch_name, - "url" => "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/new?merge_request%5Bsource_branch%5D=#{branch_name}", + "url" => "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/new?merge_request%5Bsource_branch%5D=#{branch_name}", "new_merge_request" => true }] end @@ -970,7 +970,7 @@ describe API::Internal do expect(json_response['merge_request_urls']).to match [{ 'branch_name' => branch_name, - 'url' => "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/1", + 'url' => "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/1", 'new_merge_request' => false }] end diff --git a/spec/requests/api/project_clusters_spec.rb b/spec/requests/api/project_clusters_spec.rb index a6e08ab3ab6..e8ed016db69 100644 --- a/spec/requests/api/project_clusters_spec.rb +++ b/spec/requests/api/project_clusters_spec.rb @@ -257,12 +257,22 @@ describe API::ProjectClusters do post api("/projects/#{project.id}/clusters/user", current_user), params: cluster_params end + it 'responds with 400' do + expect(response).to have_gitlab_http_status(400) + + expect(json_response['message']['base'].first).to eq('Instance does not support multiple Kubernetes clusters') + end + end + + context 'non-authorized user' do + before do + post api("/projects/#{project.id}/clusters/user", developer_user), params: cluster_params + end + it 'responds with 403' do expect(response).to have_gitlab_http_status(403) - end - it 'returns an appropriate message' do - expect(json_response['message']).to include('Instance does not support multiple Kubernetes clusters') + expect(json_response['message']).to eq('403 Forbidden') end end end diff --git a/spec/routing/environments_spec.rb b/spec/routing/environments_spec.rb index aacbe300966..28b3e79c1ff 100644 --- a/spec/routing/environments_spec.rb +++ b/spec/routing/environments_spec.rb @@ -9,7 +9,7 @@ describe 'environments routing' do end let(:environments_route) do - "#{project.namespace.name}/#{project.name}/environments/" + "#{project.full_path}/environments/" end describe 'routing environment folders' do @@ -36,13 +36,12 @@ describe 'environments routing' do end def get_folder(folder) - get("#{project.namespace.name}/#{project.name}/" \ - "environments/folders/#{folder}") + get("#{project.full_path}/environments/folders/#{folder}") end def folder_action(**opts) - options = { namespace_id: project.namespace.name, - project_id: project.name } + options = { namespace_id: project.namespace.path, + project_id: project.path } ['projects/environments#folder', options.merge(opts)] end diff --git a/spec/rubocop/cop/gitlab/rails_logger_spec.rb b/spec/rubocop/cop/gitlab/rails_logger_spec.rb new file mode 100644 index 00000000000..f0158ddcc5c --- /dev/null +++ b/spec/rubocop/cop/gitlab/rails_logger_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'rubocop' +require 'rubocop/rspec/support' +require_relative '../../../../rubocop/cop/gitlab/rails_logger' + +describe RuboCop::Cop::Gitlab::RailsLogger do + include CopHelper + + subject(:cop) { described_class.new } + + it 'flags the use of Rails.logger.error with a constant receiver' do + inspect_source("Rails.logger.error('some error')") + + expect(cop.offenses.size).to eq(1) + end + + it 'flags the use of Rails.logger.info with a constant receiver' do + inspect_source("Rails.logger.info('some info')") + + expect(cop.offenses.size).to eq(1) + end + + it 'flags the use of Rails.logger.warn with a constant receiver' do + inspect_source("Rails.logger.warn('some warning')") + + expect(cop.offenses.size).to eq(1) + end + + it 'does not flag the use of Rails.logger with a constant that is not Rails' do + inspect_source("AppLogger.error('some error')") + + expect(cop.offenses.size).to eq(0) + end + + it 'does not flag the use of logger with a send receiver' do + inspect_source("file_logger.info('important info')") + + expect(cop.offenses.size).to eq(0) + end +end diff --git a/spec/rubocop/cop/qa/element_with_pattern_spec.rb b/spec/rubocop/cop/qa/element_with_pattern_spec.rb index ef20d9a1f26..fee390caa9f 100644 --- a/spec/rubocop/cop/qa/element_with_pattern_spec.rb +++ b/spec/rubocop/cop/qa/element_with_pattern_spec.rb @@ -23,9 +23,9 @@ describe RuboCop::Cop::QA::ElementWithPattern do expect_offense(<<-RUBY) view 'app/views/shared/groups/_search_form.html.haml' do element :groups_filter, 'search_field_tag :filter' - ^^^^^^^^^^^^^^^^^^^^^^^^^^ Don't use a pattern for element, create a corresponding `qa-groups-filter` instead. + ^^^^^^^^^^^^^^^^^^^^^^^^^^ Don't use a pattern for element, create a corresponding `data-qa-selector=groups_filter` instead. element :groups_filter_placeholder, /Search by name/ - ^^^^^^^^^^^^^^ Don't use a pattern for element, create a corresponding `qa-groups-filter-placeholder` instead. + ^^^^^^^^^^^^^^ Don't use a pattern for element, create a corresponding `data-qa-selector=groups_filter_placeholder` instead. end RUBY end diff --git a/spec/serializers/environment_status_entity_spec.rb b/spec/serializers/environment_status_entity_spec.rb index f421432e8d6..cb4749f019f 100644 --- a/spec/serializers/environment_status_entity_spec.rb +++ b/spec/serializers/environment_status_entity_spec.rb @@ -70,7 +70,7 @@ describe EnvironmentStatusEntity do it 'returns metrics url' do expect(subject[:metrics_url]) - .to eq("/#{project.namespace.name}/#{project.name}/environments/#{environment.id}/deployments/#{deployment.iid}/metrics") + .to eq("/#{project.full_path}/environments/#{environment.id}/deployments/#{deployment.iid}/metrics") end end diff --git a/spec/services/audit_event_service_spec.rb b/spec/services/audit_event_service_spec.rb index 32fd98e6ef9..e42bff607b2 100644 --- a/spec/services/audit_event_service_spec.rb +++ b/spec/services/audit_event_service_spec.rb @@ -10,11 +10,8 @@ describe AuditEventService do let(:logger) { instance_double(Gitlab::AuditJsonLogger) } describe '#security_event' do - before do - expect(service).to receive(:file_logger).and_return(logger) - end - it 'creates an event and logs to a file' do + expect(service).to receive(:file_logger).and_return(logger) expect(logger).to receive(:info).with(author_id: user.id, entity_id: project.id, entity_type: "Project", @@ -22,5 +19,32 @@ describe AuditEventService do expect { service.security_event }.to change(SecurityEvent, :count).by(1) end + + it 'formats from and to fields' do + service = described_class.new( + user, project, + { + from: true, + to: false, + action: :create, + target_id: 1 + }) + expect(service).to receive(:file_logger).and_return(logger) + expect(logger).to receive(:info).with(author_id: user.id, + entity_type: 'Project', + entity_id: project.id, + from: 'true', + to: 'false', + action: :create, + target_id: 1) + + expect { service.security_event }.to change(SecurityEvent, :count).by(1) + + details = SecurityEvent.last.details + expect(details[:from]).to be true + expect(details[:to]).to be false + expect(details[:action]).to eq(:create) + expect(details[:target_id]).to eq(1) + end end end diff --git a/spec/services/boards/issues/move_service_spec.rb b/spec/services/boards/issues/move_service_spec.rb index 16e2a2fba6b..1bfb5602df2 100644 --- a/spec/services/boards/issues/move_service_spec.rb +++ b/spec/services/boards/issues/move_service_spec.rb @@ -52,5 +52,91 @@ describe Boards::Issues::MoveService do it_behaves_like 'issues move service', true end + + describe '#execute_multiple' do + set(:group) { create(:group) } + set(:user) { create(:user) } + set(:project) { create(:project, namespace: group) } + set(:board1) { create(:board, group: group) } + set(:development) { create(:group_label, group: group, name: 'Development') } + set(:testing) { create(:group_label, group: group, name: 'Testing') } + set(:list1) { create(:list, board: board1, label: development, position: 0) } + set(:list2) { create(:list, board: board1, label: testing, position: 1) } + let(:params) { { board_id: board1.id, from_list_id: list1.id, to_list_id: list2.id } } + + before do + project.add_developer(user) + end + + it 'returns false if list of issues is empty' do + expect(described_class.new(group, user, params).execute_multiple([])).to eq(false) + end + + context 'moving multiple issues' do + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + it 'moves multiple issues from one list to another' do + expect(described_class.new(group, user, params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + end + end + + context 'moving a single issue' do + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + + it 'moves one issue' do + expect(described_class.new(group, user, params).execute_multiple([issue1])).to be_truthy + + expect(issue1.labels).to eq([testing]) + end + end + + context 'moving issues visually after an existing issue' do + let(:existing_issue) { create(:labeled_issue, project: project, labels: [testing], relative_position: 10) } + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + let(:move_params) do + params.dup.tap do |hash| + hash[:move_before_id] = existing_issue.id + end + end + + it 'moves one issue' do + expect(described_class.new(group, user, move_params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + + expect(issue1.relative_position > existing_issue.relative_position).to eq(true) + expect(issue2.relative_position > issue1.relative_position).to eq(true) + end + end + + context 'moving issues visually before an existing issue' do + let(:existing_issue) { create(:labeled_issue, project: project, labels: [testing], relative_position: 10) } + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + let(:move_params) do + params.dup.tap do |hash| + hash[:move_after_id] = existing_issue.id + end + end + + it 'moves one issue' do + expect(described_class.new(group, user, move_params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + + expect(issue2.relative_position < existing_issue.relative_position).to eq(true) + expect(issue1.relative_position < issue2.relative_position).to eq(true) + end + end + end end end diff --git a/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb b/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb index be052a07da7..44407ae2793 100644 --- a/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb +++ b/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb @@ -34,6 +34,8 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService, '#execute' d stub_kubeclient_create_service_account(api_url, namespace: namespace) stub_kubeclient_create_secret(api_url, namespace: namespace) stub_kubeclient_put_secret(api_url, "#{namespace}-token", namespace: namespace) + stub_kubeclient_put_role(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, namespace: namespace) + stub_kubeclient_put_role_binding(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, namespace: namespace) stub_kubeclient_get_secret( api_url, diff --git a/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb b/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb index 382b9043566..8b874989758 100644 --- a/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb +++ b/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb @@ -143,6 +143,8 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService do stub_kubeclient_get_role_binding_error(api_url, role_binding_name, namespace: namespace) stub_kubeclient_create_role_binding(api_url, namespace: namespace) + stub_kubeclient_put_role(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, namespace: namespace) + stub_kubeclient_put_role_binding(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, namespace: namespace) end it_behaves_like 'creates service account and token' @@ -169,6 +171,24 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService do ) ) end + + it 'creates a role and role binding granting knative serving permissions to the service account' do + subject + + expect(WebMock).to have_requested(:put, api_url + "/apis/rbac.authorization.k8s.io/v1/namespaces/#{namespace}/roles/#{Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME}").with( + body: hash_including( + metadata: { + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + namespace: namespace + }, + rules: [{ + apiGroups: %w(serving.knative.dev), + resources: %w(configurations configurationgenerations routes revisions revisionuids autoscalers services), + verbs: %w(get list create update delete patch watch) + }] + ) + ) + end end end end diff --git a/spec/services/issuable/bulk_update_service_spec.rb b/spec/services/issuable/bulk_update_service_spec.rb index 3d2d4b5f216..b0bcd7a36ba 100644 --- a/spec/services/issuable/bulk_update_service_spec.rb +++ b/spec/services/issuable/bulk_update_service_spec.rb @@ -165,7 +165,7 @@ describe Issuable::BulkUpdateService do context 'when the new assignee ID is not present' do it 'does not unassign' do expect { bulk_update(issue, assignee_ids: []) } - .not_to change { issue.reload.assignees } + .not_to change(issue.assignees, :count) end end end diff --git a/spec/services/issuable/clone/content_rewriter_spec.rb b/spec/services/issuable/clone/content_rewriter_spec.rb index 230e1123280..3479c20862a 100644 --- a/spec/services/issuable/clone/content_rewriter_spec.rb +++ b/spec/services/issuable/clone/content_rewriter_spec.rb @@ -165,5 +165,18 @@ describe Issuable::Clone::ContentRewriter do expect(note.note_html).not_to eq(new_note.note_html) end end + + context "discussion notes" do + let(:note) { create(:note, noteable: original_issue, note: "sample note", project: project1) } + let!(:discussion) { create(:discussion_note_on_issue, in_reply_to: note, note: "reply to sample note") } + + it 'rewrites discussion correctly' do + subject.execute + + expect(new_issue.notes.count).to eq(original_issue.notes.count) + expect(new_issue.notes.where(discussion_id: discussion.discussion_id).count).to eq(0) + expect(original_issue.notes.where(discussion_id: discussion.discussion_id).count).to eq(1) + end + end end end diff --git a/spec/services/merge_requests/get_urls_service_spec.rb b/spec/services/merge_requests/get_urls_service_spec.rb index 0933c6d4336..9e7a5260ca4 100644 --- a/spec/services/merge_requests/get_urls_service_spec.rb +++ b/spec/services/merge_requests/get_urls_service_spec.rb @@ -8,8 +8,8 @@ describe MergeRequests::GetUrlsService do let(:project) { create(:project, :public, :repository) } let(:service) { described_class.new(project) } let(:source_branch) { "merge-test" } - let(:new_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/new?merge_request%5Bsource_branch%5D=#{source_branch}" } - let(:show_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/#{merge_request.iid}" } + let(:new_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/new?merge_request%5Bsource_branch%5D=#{source_branch}" } + let(:show_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/#{merge_request.iid}" } let(:new_branch_changes) { "#{Gitlab::Git::BLANK_SHA} 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/#{source_branch}" } let(:deleted_branch_changes) { "d14d6c0abdd253381df51a723d58691b2ee1ab08 #{Gitlab::Git::BLANK_SHA} refs/heads/#{source_branch}" } let(:existing_branch_changes) { "d14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/#{source_branch}" } @@ -119,7 +119,7 @@ describe MergeRequests::GetUrlsService do let(:new_branch_changes) { "#{Gitlab::Git::BLANK_SHA} 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/new_branch" } let(:existing_branch_changes) { "d14d6c0abdd253381df51a723d58691b2ee1ab08 570e7b2abdd848b95f2f578043fc23bd6f6fd24d refs/heads/markdown" } let(:changes) { "#{new_branch_changes}\n#{existing_branch_changes}" } - let(:new_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.namespace.name}/#{project.path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch" } + let(:new_merge_request_url) { "http://#{Gitlab.config.gitlab.host}/#{project.full_path}/merge_requests/new?merge_request%5Bsource_branch%5D=new_branch" } it 'returns 2 urls for both creating new and showing merge request' do result = service.execute(changes) diff --git a/spec/services/merge_requests/mergeability_check_service_spec.rb b/spec/services/merge_requests/mergeability_check_service_spec.rb index 6efece64092..6e827f2ea5b 100644 --- a/spec/services/merge_requests/mergeability_check_service_spec.rb +++ b/spec/services/merge_requests/mergeability_check_service_spec.rb @@ -11,7 +11,11 @@ describe MergeRequests::MergeabilityCheckService do end it 'does not change the merge ref HEAD' do - expect { subject }.not_to change(merge_request, :merge_ref_head) + merge_ref_head = merge_request.merge_ref_head + + subject + + expect(merge_request.reload.merge_ref_head).to eq merge_ref_head end it 'returns ServiceResponse.error' do @@ -73,7 +77,7 @@ describe MergeRequests::MergeabilityCheckService do it 'second call does not change the merge-ref' do expect { subject }.to change(merge_request, :merge_ref_head).from(nil) - expect { subject }.not_to change(merge_request, :merge_ref_head) + expect { subject }.not_to change(merge_request.merge_ref_head, :id) end end diff --git a/spec/services/merge_requests/update_service_spec.rb b/spec/services/merge_requests/update_service_spec.rb index f566d235787..7e5837a4798 100644 --- a/spec/services/merge_requests/update_service_spec.rb +++ b/spec/services/merge_requests/update_service_spec.rb @@ -598,7 +598,7 @@ describe MergeRequests::UpdateService, :mailer do feature_visibility_attr = :"#{merge_request.model_name.plural}_access_level" project.project_feature.update_attribute(feature_visibility_attr, ProjectFeature::PRIVATE) - expect { update_merge_request(assignee_ids: [assignee]) }.not_to change { merge_request.reload.assignees } + expect { update_merge_request(assignee_ids: [assignee]) }.not_to change(merge_request.assignees, :count) end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index f25e2fe5e2b..3e3de051732 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -215,13 +215,14 @@ describe NotificationService, :mailer do let(:project) { create(:project, :private) } let(:issue) { create(:issue, project: project, assignees: [assignee]) } let(:mentioned_issue) { create(:issue, assignees: issue.assignees) } - let(:note) { create(:note_on_issue, noteable: issue, project_id: issue.project_id, note: '@mention referenced, @unsubscribed_mentioned and @outsider also') } + let(:author) { create(:user) } + let(:note) { create(:note_on_issue, author: author, noteable: issue, project_id: issue.project_id, note: '@mention referenced, @unsubscribed_mentioned and @outsider also') } before do - build_team(note.project) + build_team(project) project.add_maintainer(issue.author) project.add_maintainer(assignee) - project.add_maintainer(note.author) + project.add_maintainer(author) @u_custom_off = create_user_with_notification(:custom, 'custom_off') project.add_guest(@u_custom_off) @@ -240,7 +241,8 @@ describe NotificationService, :mailer do describe '#new_note' do it do - add_users_with_subscription(note.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! expect(SentNotification).to receive(:record).with(issue, any_args).exactly(10).times @@ -268,7 +270,8 @@ describe NotificationService, :mailer do end it "emails the note author if they've opted into notifications about their activity" do - add_users_with_subscription(note.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! note.author.notified_of_own_activity = true @@ -415,13 +418,15 @@ describe NotificationService, :mailer do let(:project) { create(:project, :public) } let(:issue) { create(:issue, project: project, assignees: [assignee]) } let(:mentioned_issue) { create(:issue, assignees: issue.assignees) } - let(:note) { create(:note_on_issue, noteable: issue, project_id: issue.project_id, note: '@all mentioned') } + let(:author) { create(:user) } + let(:note) { create(:note_on_issue, author: author, noteable: issue, project_id: issue.project_id, note: '@all mentioned') } before do - build_team(note.project) - build_group(note.project) - note.project.add_maintainer(note.author) - add_users_with_subscription(note.project, issue) + build_team(project) + build_group(project) + add_users(project) + add_user_subscriptions(issue) + project.add_maintainer(author) reset_delivered_emails! end @@ -473,17 +478,18 @@ describe NotificationService, :mailer do context 'project snippet note' do let!(:project) { create(:project, :public) } let(:snippet) { create(:project_snippet, project: project, author: create(:user)) } - let(:note) { create(:note_on_project_snippet, noteable: snippet, project_id: project.id, note: '@all mentioned') } + let(:author) { create(:user) } + let(:note) { create(:note_on_project_snippet, author: author, noteable: snippet, project_id: project.id, note: '@all mentioned') } before do build_team(project) build_group(project) + project.add_maintainer(author) # make sure these users can read the project snippet! project.add_guest(@u_guest_watcher) project.add_guest(@u_guest_custom) add_member_for_parent_group(@pg_watcher, project) - note.project.add_maintainer(note.author) reset_delivered_emails! end @@ -708,10 +714,11 @@ describe NotificationService, :mailer do let(:issue) { create :issue, project: project, assignees: [assignee], description: 'cc @participant @unsubscribed_mentioned' } before do - build_team(issue.project) - build_group(issue.project) + build_team(project) + build_group(project) - add_users_with_subscription(issue.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! update_custom_notification(:new_issue, @u_guest_custom, resource: project) update_custom_notification(:new_issue, @u_custom_global) @@ -1281,13 +1288,16 @@ describe NotificationService, :mailer do let(:project) { create(:project, :public, :repository, namespace: group) } let(:another_project) { create(:project, :public, namespace: group) } let(:assignee) { create(:user) } - let(:merge_request) { create :merge_request, source_project: project, assignees: [assignee], description: 'cc @participant' } + let(:assignees) { Array.wrap(assignee) } + let(:author) { create(:user) } + let(:merge_request) { create :merge_request, author: author, source_project: project, assignees: assignees, description: 'cc @participant' } before do - project.add_maintainer(merge_request.author) - merge_request.assignees.each { |assignee| project.add_maintainer(assignee) } - build_team(merge_request.target_project) - add_users_with_subscription(merge_request.target_project, merge_request) + project.add_maintainer(author) + assignees.each { |assignee| project.add_maintainer(assignee) } + build_team(project) + add_users(project) + add_user_subscriptions(merge_request) update_custom_notification(:new_merge_request, @u_guest_custom, resource: project) update_custom_notification(:new_merge_request, @u_custom_global) reset_delivered_emails! @@ -1834,7 +1844,7 @@ describe NotificationService, :mailer do describe 'ProjectMember' do let(:project) { create(:project) } - set(:added_user) { create(:user) } + let(:added_user) { create(:user) } describe '#new_access_request' do context 'for a project in a user namespace' do @@ -2417,7 +2427,7 @@ describe NotificationService, :mailer do should_not_email(user, recipients: email_recipients) end - def add_users_with_subscription(project, issuable) + def add_users(project) @subscriber = create :user @unsubscriber = create :user @unsubscribed_mentioned = create :user, username: 'unsubscribed_mentioned' @@ -2429,7 +2439,9 @@ describe NotificationService, :mailer do project.add_maintainer(@unsubscriber) project.add_maintainer(@watcher_and_subscriber) project.add_maintainer(@unsubscribed_mentioned) + end + def add_user_subscriptions(issuable) issuable.subscriptions.create(user: @unsubscribed_mentioned, project: project, subscribed: false) issuable.subscriptions.create(user: @subscriber, project: project, subscribed: true) issuable.subscriptions.create(user: @subscribed_participant, project: project, subscribed: true) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 62fdc039b5e..95e0d8858b9 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -47,7 +47,7 @@ Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f } quality_level = Quality::TestLevel.new RSpec.configure do |config| - config.use_transactional_fixtures = false + config.use_transactional_fixtures = true config.use_instantiated_fixtures = false config.fixture_path = Rails.root @@ -103,6 +103,7 @@ RSpec.configure do |config| config.include RedisHelpers config.include Rails.application.routes.url_helpers, type: :routing config.include PolicyHelpers, type: :policy + config.include MemoryUsageHelper if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. @@ -289,6 +290,16 @@ RSpec.configure do |config| config.before(:each, :https_pages_disabled) do |_| allow(Gitlab.config.pages).to receive(:external_https).and_return(false) end + + # We can't use an `around` hook here because the wrapping transaction + # is not yet opened at the time that is triggered + config.prepend_before do + Gitlab::Database.set_open_transactions_baseline + end + + config.append_after do + Gitlab::Database.reset_open_transactions_baseline + end end # add simpler way to match asset paths containing digest strings diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index edd7de94203..f0dd6c52b74 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -26,31 +26,22 @@ RSpec.configure do |config| end config.append_after(:context) do - DatabaseCleaner.clean_with(:deletion, cache_tables: false) + delete_from_all_tables! end - config.before do - setup_database_cleaner - DatabaseCleaner.strategy = :transaction - end + config.around(:each, :delete) do |example| + self.class.use_transactional_tests = false - config.before(:each, :js) do - DatabaseCleaner.strategy = :deletion, { except: deletion_except_tables, cache_tables: false } - end + example.run - config.before(:each, :delete) do - DatabaseCleaner.strategy = :deletion, { except: deletion_except_tables, cache_tables: false } + delete_from_all_tables!(except: deletion_except_tables) end - config.before(:each, :migration) do - DatabaseCleaner.strategy = :deletion, { cache_tables: false } - end + config.around(:each, :migration) do |example| + self.class.use_transactional_tests = false - config.before do - DatabaseCleaner.start - end + example.run - config.append_after do - DatabaseCleaner.clean + delete_from_all_tables! end end diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index c69fa322073..08622dff6d9 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -1,4 +1,8 @@ module DbCleaner + def delete_from_all_tables!(except: nil) + DatabaseCleaner.clean_with(:deletion, cache_tables: false, except: except) + end + def deletion_except_tables [] end diff --git a/spec/support/features/rss_shared_examples.rb b/spec/support/features/rss_shared_examples.rb index 0de92aedba5..02d310a9afa 100644 --- a/spec/support/features/rss_shared_examples.rb +++ b/spec/support/features/rss_shared_examples.rb @@ -6,7 +6,7 @@ end shared_examples "it has an RSS button with current_user's feed token" do it "shows the RSS button with current_user's feed token" do - expect(page).to have_css("a:has(.fa-rss)[href*='feed_token=#{user.feed_token}']") + expect(page).to have_css("a:has(.fa-rss)[href*='feed_token=#{user.feed_token}'], .js-rss-button[href*='feed_token=#{user.feed_token}']") end end @@ -18,6 +18,6 @@ end shared_examples "it has an RSS button without a feed token" do it "shows the RSS button without a feed token" do - expect(page).to have_css("a:has(.fa-rss):not([href*='feed_token'])") + expect(page).to have_css("a:has(.fa-rss):not([href*='feed_token']), .js-rss-button:not([href*='feed_token'])") end end diff --git a/spec/support/helpers/kubernetes_helpers.rb b/spec/support/helpers/kubernetes_helpers.rb index 3c7bcba2b42..278264f3df5 100644 --- a/spec/support/helpers/kubernetes_helpers.rb +++ b/spec/support/helpers/kubernetes_helpers.rb @@ -199,6 +199,11 @@ module KubernetesHelpers .to_return(kube_response({})) end + def stub_kubeclient_put_role(api_url, name, namespace: 'default') + WebMock.stub_request(:put, api_url + "/apis/rbac.authorization.k8s.io/v1/namespaces/#{namespace}/roles/#{name}") + .to_return(kube_response({})) + end + def kube_v1_secret_body(**options) { "kind" => "SecretList", diff --git a/spec/support/helpers/memory_usage_helper.rb b/spec/support/helpers/memory_usage_helper.rb new file mode 100644 index 00000000000..984ea8cc571 --- /dev/null +++ b/spec/support/helpers/memory_usage_helper.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module MemoryUsageHelper + extend ActiveSupport::Concern + + def gather_memory_data(csv_path) + write_csv_entry(csv_path, + { + example_group_path: TestEnv.topmost_example_group[:location], + example_group_description: TestEnv.topmost_example_group[:description], + time: Time.current, + job_name: ENV['CI_JOB_NAME'] + }.merge(get_memory_usage)) + end + + def write_csv_entry(path, entry) + CSV.open(path, "a", headers: entry.keys, write_headers: !File.exist?(path)) do |file| + file << entry.values + end + end + + def get_memory_usage + output, status = Gitlab::Popen.popen(%w(free -m)) + abort "`free -m` return code is #{status}: #{output}" unless status.zero? + + result = output.split("\n")[1].split(" ")[1..-1] + attrs = %i(m_total m_used m_free m_shared m_buffers_cache m_available).freeze + + attrs.zip(result).to_h + end + + included do |config| + config.after(:all) do + gather_memory_data(ENV['MEMORY_TEST_PATH']) if ENV['MEMORY_TEST_PATH'] + end + end +end diff --git a/spec/support/helpers/migrations_helpers.rb b/spec/support/helpers/migrations_helpers.rb index cc1a28cb264..272b24f7541 100644 --- a/spec/support/helpers/migrations_helpers.rb +++ b/spec/support/helpers/migrations_helpers.rb @@ -18,8 +18,12 @@ module MigrationsHelpers ActiveRecord::Migrator.migrations_paths end + def migration_context + ActiveRecord::MigrationContext.new(migrations_paths) + end + def migrations - ActiveRecord::Migrator.migrations(migrations_paths) + migration_context.migrations end def clear_schema_cache! @@ -96,8 +100,7 @@ module MigrationsHelpers def schema_migrate_down! disable_migrations_output do - ActiveRecord::Migrator.migrate(migrations_paths, - migration_schema_version) + migration_context.down(migration_schema_version) end reset_column_in_all_models @@ -107,7 +110,7 @@ module MigrationsHelpers reset_column_in_all_models disable_migrations_output do - ActiveRecord::Migrator.migrate(migrations_paths) + migration_context.up end reset_column_in_all_models @@ -123,7 +126,7 @@ module MigrationsHelpers end def migrate! - ActiveRecord::Migrator.up(migrations_paths) do |migration| + migration_context.up do |migration| migration.name == described_class.name end end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index e63099d89b7..893b10ea752 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -2,6 +2,7 @@ require 'rspec/mocks' require 'toml-rb' module TestEnv + extend ActiveSupport::Concern extend self ComponentFailedToInstallError = Class.new(StandardError) @@ -108,6 +109,12 @@ module TestEnv setup_forked_repo end + included do |config| + config.append_before do + set_current_example_group + end + end + def disable_mailer allow_any_instance_of(NotificationService).to receive(:mailer) .and_return(double.as_null_object) @@ -297,8 +304,23 @@ module TestEnv FileUtils.rm_rf(path) end + def current_example_group + Thread.current[:current_example_group] + end + + # looking for a top-level `describe` + def topmost_example_group + example_group = current_example_group + example_group = example_group[:parent_example_group] until example_group[:parent_example_group].nil? + example_group + end + private + def set_current_example_group + Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group] + end + # These are directories that should be preserved at cleanup time def test_dirs @test_dirs ||= %w[ diff --git a/spec/support/matchers/abort_matcher.rb b/spec/support/matchers/abort_matcher.rb new file mode 100644 index 00000000000..ce1dd140210 --- /dev/null +++ b/spec/support/matchers/abort_matcher.rb @@ -0,0 +1,46 @@ +RSpec::Matchers.define :abort_execution do + match do |code_block| + @captured_stderr = StringIO.new + original_stderr = $stderr + $stderr = @captured_stderr + + code_block.call + + false + rescue SystemExit => e + captured = @captured_stderr.string.chomp + @actual_exit_code = e.status + break false unless e.status == 1 + + if @message + if @message.is_a? String + @message == captured + elsif @message.is_a? Regexp + @message.match?(captured) + else + raise ArgumentError, 'with_message must be either a String or a Regular Expression' + end + end + + ensure + $stderr = original_stderr + end + + chain :with_message do |message| + @message = message + end + + failure_message do |block| + unless @actual_exit_code + break "expected #{block} to abort with '#{@message}' but didnt call abort." + end + + if @actual_exit_code != 1 + break "expected #{block} to abort with: '#{@message}' but exited with success instead." + end + + "expected #{block} to abort with: '#{@message}' \n but received: '#{@captured_stderr.string.chomp}' instead." + end + + supports_block_expectations +end diff --git a/spec/support/shared_contexts/email_shared_context.rb b/spec/support/shared_contexts/email_shared_context.rb index 9d806fc524d..4f5d53f9317 100644 --- a/spec/support/shared_contexts/email_shared_context.rb +++ b/spec/support/shared_contexts/email_shared_context.rb @@ -1,5 +1,3 @@ -require 'gitlab/email/receiver' - shared_context :email_shared_context do let(:mail_key) { "59d8df8370b7e95c5a49fbf86aeb2c93" } let(:receiver) { Gitlab::Email::Receiver.new(email_raw) } diff --git a/spec/support/shared_examples/mentionable_shared_examples.rb b/spec/support/shared_examples/mentionable_shared_examples.rb index 1226841f24c..fea52c2eeb2 100644 --- a/spec/support/shared_examples/mentionable_shared_examples.rb +++ b/spec/support/shared_examples/mentionable_shared_examples.rb @@ -76,6 +76,30 @@ shared_examples 'a mentionable' do expect(refs).to include(ext_commit) end + context 'when there are cached markdown fields' do + before do + if subject.is_a?(CacheMarkdownField) + subject.refresh_markdown_cache + end + end + + it 'sends in cached markdown fields when appropriate' do + if subject.is_a?(CacheMarkdownField) + expect_next_instance_of(Gitlab::ReferenceExtractor) do |ext| + attrs = subject.class.mentionable_attrs.collect(&:first) & subject.cached_markdown_fields.markdown_fields + attrs.each do |field| + expect(ext).to receive(:analyze).with(subject.send(field), hash_including(rendered: anything)) + end + end + + expect(subject).not_to receive(:refresh_markdown_cache) + expect(subject).to receive(:cached_markdown_fields).at_least(:once).and_call_original + + subject.all_references(author) + end + end + end + it 'creates cross-reference notes' do mentioned_objects = [mentioned_issue, mentioned_mr, mentioned_commit, ext_issue, ext_mr, ext_commit] @@ -98,6 +122,33 @@ shared_examples 'an editable mentionable' do [create(:issue, project: project), create(:issue, project: ext_proj)] end + context 'when there are cached markdown fields' do + before do + if subject.is_a?(CacheMarkdownField) + subject.refresh_markdown_cache + end + end + + it 'refreshes markdown cache if necessary' do + subject.save! + + set_mentionable_text.call('This is a text') + + if subject.is_a?(CacheMarkdownField) + expect_next_instance_of(Gitlab::ReferenceExtractor) do |ext| + subject.cached_markdown_fields.markdown_fields.each do |field| + expect(ext).to receive(:analyze).with(subject.send(field), hash_including(rendered: anything)) + end + end + + expect(subject).to receive(:refresh_markdown_cache) + expect(subject).to receive(:cached_markdown_fields).at_least(:once).and_call_original + + subject.all_references(author) + end + end + end + it 'creates new cross-reference notes when the mentionable text is edited' do subject.save subject.create_cross_references! diff --git a/spec/support/shared_examples/policies/clusterable_shared_examples.rb b/spec/support/shared_examples/policies/clusterable_shared_examples.rb index d99f94c76c3..4f9873d53e4 100644 --- a/spec/support/shared_examples/policies/clusterable_shared_examples.rb +++ b/spec/support/shared_examples/policies/clusterable_shared_examples.rb @@ -24,14 +24,6 @@ shared_examples 'clusterable policies' do context 'with no clusters' do it { expect_allowed(:add_cluster) } end - - context 'with an existing cluster' do - before do - cluster - end - - it { expect_disallowed(:add_cluster) } - end end end end diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 4b04d9cec39..0e47408fc72 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -50,7 +50,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/This task requires database write access. Exiting./) end end end @@ -96,7 +96,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/There is already a rollback operation in progress/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a rollback operation in progress/) end end end @@ -105,14 +105,23 @@ describe 'rake gitlab:storage:*', :sidekiq do it 'does nothing' do expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async) - run_rake_task(task) + expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects requiring storage migration. Nothing to do!') end end context 'with 3 legacy projects' do let(:projects) { create_list(:project, 3, :legacy_storage) } - it_behaves_like "handles custom BATCH env var", ::HashedStorage::MigratorWorker + it 'enqueues migrations and count projects correctly' do + projects.map(&:id).sort.tap do |ids| + stub_env('ID_FROM', ids[0]) + stub_env('ID_TO', ids[1]) + end + + expect { run_rake_task(task) }.to output(/Enqueuing migration of 2 projects in batches/).to_stdout + end + + it_behaves_like 'handles custom BATCH env var', ::HashedStorage::MigratorWorker end context 'with same id in range' do @@ -120,7 +129,7 @@ describe 'rake gitlab:storage:*', :sidekiq do stub_env('ID_FROM', 99999) stub_env('ID_TO', 99999) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=99999/) end it 'displays a message when project exists but its already migrated' do @@ -128,7 +137,7 @@ describe 'rake gitlab:storage:*', :sidekiq do stub_env('ID_FROM', project.id) stub_env('ID_TO', project.id) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=#{project.id}/) end it 'enqueues migration when project can be found' do @@ -153,7 +162,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/There is already a migration operation in progress/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a migration operation in progress/) end end end @@ -162,13 +171,22 @@ describe 'rake gitlab:storage:*', :sidekiq do it 'does nothing' do expect(::HashedStorage::RollbackerWorker).not_to receive(:perform_async) - run_rake_task(task) + expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects that can have storage rolledback. Nothing to do!') end end context 'with 3 hashed projects' do let(:projects) { create_list(:project, 3) } + it 'enqueues migrations and count projects correctly' do + projects.map(&:id).sort.tap do |ids| + stub_env('ID_FROM', ids[0]) + stub_env('ID_TO', ids[1]) + end + + expect { run_rake_task(task) }.to output(/Enqueuing rollback of 2 projects in batches/).to_stdout + end + it_behaves_like "handles custom BATCH env var", ::HashedStorage::RollbackerWorker end end diff --git a/spec/views/notify/pipeline_failed_email.html.haml_spec.rb b/spec/views/notify/pipeline_failed_email.html.haml_spec.rb index 623237b111a..bf633a118ca 100644 --- a/spec/views/notify/pipeline_failed_email.html.haml_spec.rb +++ b/spec/views/notify/pipeline_failed_email.html.haml_spec.rb @@ -28,7 +28,7 @@ describe 'notify/pipeline_failed_email.html.haml' do expect(rendered).to have_content "Your pipeline has failed" expect(rendered).to have_content pipeline.project.name - expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub!(/\s+/, ' ') + expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub(/\s+/, ' ') expect(rendered).to have_content pipeline.commit.author_name expect(rendered).to have_content "##{pipeline.id}" expect(rendered).to have_content pipeline.user.name @@ -45,7 +45,7 @@ describe 'notify/pipeline_failed_email.html.haml' do expect(rendered).to have_content "Your pipeline has failed" expect(rendered).to have_content pipeline.project.name - expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub!(/\s+/, ' ') + expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub(/\s+/, ' ') expect(rendered).to have_content pipeline.commit.author_name expect(rendered).to have_content "##{pipeline.id}" expect(rendered).to have_content "by API" diff --git a/spec/views/notify/pipeline_failed_email.text.erb_spec.rb b/spec/views/notify/pipeline_failed_email.text.erb_spec.rb index 81245239eba..060274eb56a 100644 --- a/spec/views/notify/pipeline_failed_email.text.erb_spec.rb +++ b/spec/views/notify/pipeline_failed_email.text.erb_spec.rb @@ -30,7 +30,7 @@ describe 'notify/pipeline_failed_email.text.erb' do expect(rendered).to have_content('Your pipeline has failed') expect(rendered).to have_content(pipeline.project.name) - expect(rendered).to have_content(pipeline.git_commit_message.truncate(50).gsub!(/\s+/, ' ')) + expect(rendered).to have_content(pipeline.git_commit_message.truncate(50).gsub(/\s+/, ' ')) expect(rendered).to have_content(pipeline.commit.author_name) expect(rendered).to have_content("##{pipeline.id}") expect(rendered).to have_content(pipeline.user.name) diff --git a/spec/views/notify/pipeline_success_email.html.haml_spec.rb b/spec/views/notify/pipeline_success_email.html.haml_spec.rb index a876bf13e11..46a6c908863 100644 --- a/spec/views/notify/pipeline_success_email.html.haml_spec.rb +++ b/spec/views/notify/pipeline_success_email.html.haml_spec.rb @@ -28,7 +28,7 @@ describe 'notify/pipeline_success_email.html.haml' do expect(rendered).to have_content "Your pipeline has passed" expect(rendered).to have_content pipeline.project.name - expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub!(/\s+/, ' ') + expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub(/\s+/, ' ') expect(rendered).to have_content pipeline.commit.author_name expect(rendered).to have_content "##{pipeline.id}" expect(rendered).to have_content pipeline.user.name @@ -45,7 +45,7 @@ describe 'notify/pipeline_success_email.html.haml' do expect(rendered).to have_content "Your pipeline has passed" expect(rendered).to have_content pipeline.project.name - expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub!(/\s+/, ' ') + expect(rendered).to have_content pipeline.git_commit_message.truncate(50).gsub(/\s+/, ' ') expect(rendered).to have_content pipeline.commit.author_name expect(rendered).to have_content "##{pipeline.id}" expect(rendered).to have_content "by API" diff --git a/yarn.lock b/yarn.lock index dc5e0662396..e8c11b1a033 100644 --- a/yarn.lock +++ b/yarn.lock @@ -705,10 +705,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.67.0.tgz#c7b94eca13b99fd3aaa737fb6dcc0abc41d3c579" integrity sha512-hJOmWEs6RkjzyKkb1vc9wwKGZIBIP0coHkxu/KgOoxhBVudpGk4CH7xJ6UuB2TKpb0SEh5CC1CzRZfBYaFhsaA== -"@gitlab/ui@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.5.0.tgz#2000b2ed0c3825dd8c4430191023f4d03c923ecb" - integrity sha512-6e/AFFLDk/gm4wKnHM9rcpTRqCsWkPKW/Vjsnd6h4wyfif2/TusHeIn/jedQxUaORbO/XZKzg4V5COhXXbCx4w== +"@gitlab/ui@^5.6.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.6.0.tgz#6b5408050229e2135359f3fce5d0de7718326a0d" + integrity sha512-EohpACc5OCK8pOWgwB7/QZUcg3tA9k863ku6Ik9NxaRSKt/JIpQ8RI4wCr4UmqhejZLQMD9VZHLUmc9Sf3Mk9w== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" |