summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-06-25 00:09:26 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-06-25 00:09:26 +0000
commit5d6119a1a4d31009ea7aee400c8462e897ddc26c (patch)
treeb648d64a6cbcb9195325a258715ba23633f88984
parentd627a16f3d10e2188ae2b2797e042f4c324718db (diff)
downloadgitlab-ce-5d6119a1a4d31009ea7aee400c8462e897ddc26c.tar.gz
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--CHANGELOG-EE.md4
-rw-r--r--app/helpers/ci/status_helper.rb148
-rw-r--r--app/helpers/ci_status_helper.rb146
-rw-r--r--app/helpers/deploy_tokens_helper.rb10
-rw-r--r--app/views/shared/deploy_tokens/_form.html.haml1
-rw-r--r--changelogs/unreleased/ab-partition-management.yml5
-rw-r--r--config/initializers/active_record_schema_ignore_tables.rb3
-rw-r--r--db/migrate/20200623121135_create_dynamic_partitions_schema.rb19
-rw-r--r--db/structure.sql5
-rw-r--r--doc/administration/monitoring/performance/performance_bar.md3
-rw-r--r--doc/development/rake_tasks.md3
-rw-r--r--lib/backup/database.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/records_fetcher.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb8
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb8
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb7
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb23
-rw-r--r--lib/gitlab/database.rb7
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb2
-rw-r--r--lib/gitlab/database/schema_helpers.rb6
-rw-r--r--lib/tasks/gitlab/db.rake5
-rw-r--r--locale/gitlab.pot3
-rw-r--r--spec/db/schema_spec.rb30
-rw-r--r--spec/helpers/ci/status_helper_spec.rb (renamed from spec/helpers/ci_status_helper_spec.rb)2
-rw-r--r--spec/lib/backup/database_spec.rb67
-rw-r--r--spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb2
-rw-r--r--spec/lib/gitlab/database_spec.rb8
-rw-r--r--spec/support/helpers/partitioning_helpers.rb9
-rw-r--r--spec/support/shared_examples/lib/gitlab/cycle_analytics/event_shared_examples.rb1
31 files changed, 385 insertions, 172 deletions
diff --git a/CHANGELOG-EE.md b/CHANGELOG-EE.md
index d7bdd181997..bed54e25ea9 100644
--- a/CHANGELOG-EE.md
+++ b/CHANGELOG-EE.md
@@ -354,6 +354,10 @@ Please view this file on the master branch, on stable branches it's out of date.
- Translate unauthenticated user string for Audit Event. !31856 (Sashi Kumar)
+## 12.10.12 (2020-06-24)
+
+- No changes.
+
## 12.10.11 (2020-06-10)
### Security (1 change)
diff --git a/app/helpers/ci/status_helper.rb b/app/helpers/ci/status_helper.rb
new file mode 100644
index 00000000000..bca49324a19
--- /dev/null
+++ b/app/helpers/ci/status_helper.rb
@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+##
+# DEPRECATED
+#
+# These helpers are deprecated in favor of detailed CI/CD statuses.
+#
+# See 'detailed_status?` method and `Gitlab::Ci::Status` module.
+#
+module Ci
+ module StatusHelper
+ def ci_label_for_status(status)
+ if detailed_status?(status)
+ return status.label
+ end
+
+ label = case status
+ when 'success'
+ 'passed'
+ when 'success-with-warnings'
+ 'passed with warnings'
+ when 'manual'
+ 'waiting for manual action'
+ when 'scheduled'
+ 'waiting for delayed job'
+ else
+ status
+ end
+ translation = "CiStatusLabel|#{label}"
+ s_(translation)
+ end
+
+ def ci_text_for_status(status)
+ if detailed_status?(status)
+ return status.text
+ end
+
+ case status
+ when 'success'
+ s_('CiStatusText|passed')
+ when 'success-with-warnings'
+ s_('CiStatusText|passed')
+ when 'manual'
+ s_('CiStatusText|blocked')
+ when 'scheduled'
+ s_('CiStatusText|delayed')
+ else
+ # All states are already being translated inside the detailed statuses:
+ # :running => Gitlab::Ci::Status::Running
+ # :skipped => Gitlab::Ci::Status::Skipped
+ # :failed => Gitlab::Ci::Status::Failed
+ # :success => Gitlab::Ci::Status::Success
+ # :canceled => Gitlab::Ci::Status::Canceled
+ # The following states are customized above:
+ # :manual => Gitlab::Ci::Status::Manual
+ status_translation = "CiStatusText|#{status}"
+ s_(status_translation)
+ end
+ end
+
+ def ci_status_for_statuseable(subject)
+ status = subject.try(:status) || 'not found'
+ status.humanize
+ end
+
+ # rubocop:disable Metrics/CyclomaticComplexity
+ def ci_icon_for_status(status, size: 16)
+ if detailed_status?(status)
+ return sprite_icon(status.icon, size: size)
+ end
+
+ icon_name =
+ case status
+ when 'success'
+ 'status_success'
+ when 'success-with-warnings'
+ 'status_warning'
+ when 'failed'
+ 'status_failed'
+ when 'pending'
+ 'status_pending'
+ when 'waiting_for_resource'
+ 'status_pending'
+ when 'preparing'
+ 'status_preparing'
+ when 'running'
+ 'status_running'
+ when 'play'
+ 'play'
+ when 'created'
+ 'status_created'
+ when 'skipped'
+ 'status_skipped'
+ when 'manual'
+ 'status_manual'
+ when 'scheduled'
+ 'status_scheduled'
+ else
+ 'status_canceled'
+ end
+
+ sprite_icon(icon_name, size: size)
+ end
+ # rubocop:enable Metrics/CyclomaticComplexity
+
+ def ci_icon_class_for_status(status)
+ group = detailed_status?(status) ? status.group : status.dasherize
+
+ "ci-status-icon-#{group}"
+ end
+
+ def pipeline_status_cache_key(pipeline_status)
+ "pipeline-status/#{pipeline_status.sha}-#{pipeline_status.status}"
+ end
+
+ def render_commit_status(commit, status, ref: nil, tooltip_placement: 'left')
+ project = commit.project
+ path = pipelines_project_commit_path(project, commit, ref: ref)
+
+ render_status_with_link(
+ status,
+ path,
+ tooltip_placement: tooltip_placement,
+ icon_size: 24)
+ end
+
+ def render_status_with_link(status, path = nil, type: _('pipeline'), tooltip_placement: 'left', cssclass: '', container: 'body', icon_size: 16)
+ klass = "ci-status-link #{ci_icon_class_for_status(status)} d-inline-flex #{cssclass}"
+ title = "#{type.titleize}: #{ci_label_for_status(status)}"
+ data = { toggle: 'tooltip', placement: tooltip_placement, container: container }
+
+ if path
+ link_to ci_icon_for_status(status, size: icon_size), path,
+ class: klass, title: title, data: data
+ else
+ content_tag :span, ci_icon_for_status(status, size: icon_size),
+ class: klass, title: title, data: data
+ end
+ end
+
+ def detailed_status?(status)
+ status.respond_to?(:text) &&
+ status.respond_to?(:group) &&
+ status.respond_to?(:label) &&
+ status.respond_to?(:icon)
+ end
+ end
+end
diff --git a/app/helpers/ci_status_helper.rb b/app/helpers/ci_status_helper.rb
deleted file mode 100644
index 80d1b7e7edb..00000000000
--- a/app/helpers/ci_status_helper.rb
+++ /dev/null
@@ -1,146 +0,0 @@
-# frozen_string_literal: true
-
-##
-# DEPRECATED
-#
-# These helpers are deprecated in favor of detailed CI/CD statuses.
-#
-# See 'detailed_status?` method and `Gitlab::Ci::Status` module.
-#
-module CiStatusHelper
- def ci_label_for_status(status)
- if detailed_status?(status)
- return status.label
- end
-
- label = case status
- when 'success'
- 'passed'
- when 'success-with-warnings'
- 'passed with warnings'
- when 'manual'
- 'waiting for manual action'
- when 'scheduled'
- 'waiting for delayed job'
- else
- status
- end
- translation = "CiStatusLabel|#{label}"
- s_(translation)
- end
-
- def ci_text_for_status(status)
- if detailed_status?(status)
- return status.text
- end
-
- case status
- when 'success'
- s_('CiStatusText|passed')
- when 'success-with-warnings'
- s_('CiStatusText|passed')
- when 'manual'
- s_('CiStatusText|blocked')
- when 'scheduled'
- s_('CiStatusText|delayed')
- else
- # All states are already being translated inside the detailed statuses:
- # :running => Gitlab::Ci::Status::Running
- # :skipped => Gitlab::Ci::Status::Skipped
- # :failed => Gitlab::Ci::Status::Failed
- # :success => Gitlab::Ci::Status::Success
- # :canceled => Gitlab::Ci::Status::Canceled
- # The following states are customized above:
- # :manual => Gitlab::Ci::Status::Manual
- status_translation = "CiStatusText|#{status}"
- s_(status_translation)
- end
- end
-
- def ci_status_for_statuseable(subject)
- status = subject.try(:status) || 'not found'
- status.humanize
- end
-
- # rubocop:disable Metrics/CyclomaticComplexity
- def ci_icon_for_status(status, size: 16)
- if detailed_status?(status)
- return sprite_icon(status.icon, size: size)
- end
-
- icon_name =
- case status
- when 'success'
- 'status_success'
- when 'success-with-warnings'
- 'status_warning'
- when 'failed'
- 'status_failed'
- when 'pending'
- 'status_pending'
- when 'waiting_for_resource'
- 'status_pending'
- when 'preparing'
- 'status_preparing'
- when 'running'
- 'status_running'
- when 'play'
- 'play'
- when 'created'
- 'status_created'
- when 'skipped'
- 'status_skipped'
- when 'manual'
- 'status_manual'
- when 'scheduled'
- 'status_scheduled'
- else
- 'status_canceled'
- end
-
- sprite_icon(icon_name, size: size)
- end
- # rubocop:enable Metrics/CyclomaticComplexity
-
- def ci_icon_class_for_status(status)
- group = detailed_status?(status) ? status.group : status.dasherize
-
- "ci-status-icon-#{group}"
- end
-
- def pipeline_status_cache_key(pipeline_status)
- "pipeline-status/#{pipeline_status.sha}-#{pipeline_status.status}"
- end
-
- def render_commit_status(commit, status, ref: nil, tooltip_placement: 'left')
- project = commit.project
- path = pipelines_project_commit_path(project, commit, ref: ref)
-
- render_status_with_link(
- status,
- path,
- tooltip_placement: tooltip_placement,
- icon_size: 24)
- end
-
- def render_status_with_link(status, path = nil, type: _('pipeline'), tooltip_placement: 'left', cssclass: '', container: 'body', icon_size: 16)
- klass = "ci-status-link #{ci_icon_class_for_status(status)} d-inline-flex #{cssclass}"
- title = "#{type.titleize}: #{ci_label_for_status(status)}"
- data = { toggle: 'tooltip', placement: tooltip_placement, container: container }
-
- if path
- link_to ci_icon_for_status(status, size: icon_size), path,
- class: klass, title: title, data: data
- else
- content_tag :span, ci_icon_for_status(status, size: icon_size),
- class: klass, title: title, data: data
- end
- end
-
- def detailed_status?(status)
- status.respond_to?(:text) &&
- status.respond_to?(:group) &&
- status.respond_to?(:label) &&
- status.respond_to?(:icon)
- end
-end
diff --git a/app/helpers/deploy_tokens_helper.rb b/app/helpers/deploy_tokens_helper.rb
index eeeeb14d991..80a5bb44c69 100644
--- a/app/helpers/deploy_tokens_helper.rb
+++ b/app/helpers/deploy_tokens_helper.rb
@@ -7,14 +7,8 @@ module DeployTokensHelper
Rails.env.test?
end
- def container_registry_enabled?(subject)
+ def container_registry_enabled?(project)
Gitlab.config.registry.enabled &&
- can?(current_user, :read_container_image, subject)
- end
-
- def packages_registry_enabled?(subject)
- Gitlab.config.packages.enabled &&
- subject.feature_available?(:packages) &&
- can?(current_user, :read_package, subject)
+ can?(current_user, :read_container_image, project)
end
end
diff --git a/app/views/shared/deploy_tokens/_form.html.haml b/app/views/shared/deploy_tokens/_form.html.haml
index 00e1cc4e9ca..512644518fa 100644
--- a/app/views/shared/deploy_tokens/_form.html.haml
+++ b/app/views/shared/deploy_tokens/_form.html.haml
@@ -35,7 +35,6 @@
= label_tag ("deploy_token_write_registry"), 'write_registry', class: 'label-bold form-check-label'
.text-secondary= s_('DeployTokens|Allows write access to the registry images')
- - if packages_registry_enabled?(group_or_project)
%fieldset.form-group.form-check
= f.check_box :read_package_registry, class: 'form-check-input'
= label_tag ("deploy_token_read_package_registry"), 'read_package_registry', class: 'label-bold form-check-label'
diff --git a/changelogs/unreleased/ab-partition-management.yml b/changelogs/unreleased/ab-partition-management.yml
new file mode 100644
index 00000000000..c4c9c8ab432
--- /dev/null
+++ b/changelogs/unreleased/ab-partition-management.yml
@@ -0,0 +1,5 @@
+---
+title: Create time-space partitions in separate schema gitlab_partitions_dynamic
+merge_request: 35137
+author:
+type: other
diff --git a/config/initializers/active_record_schema_ignore_tables.rb b/config/initializers/active_record_schema_ignore_tables.rb
index 661135f8ade..8ac565f239e 100644
--- a/config/initializers/active_record_schema_ignore_tables.rb
+++ b/config/initializers/active_record_schema_ignore_tables.rb
@@ -1,2 +1,5 @@
# Ignore table used temporarily in background migration
ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"]
+
+# Ignore dynamically managed partitions in static application schema
+ActiveRecord::SchemaDumper.ignore_tables += ["#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.*"]
diff --git a/db/migrate/20200623121135_create_dynamic_partitions_schema.rb b/db/migrate/20200623121135_create_dynamic_partitions_schema.rb
new file mode 100644
index 00000000000..931a55ebcf4
--- /dev/null
+++ b/db/migrate/20200623121135_create_dynamic_partitions_schema.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateDynamicPartitionsSchema < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::SchemaHelpers
+
+ DOWNTIME = false
+
+ def up
+ execute 'CREATE SCHEMA gitlab_partitions_dynamic'
+
+ create_comment(:schema, :gitlab_partitions_dynamic, <<~EOS.strip)
+ Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.
+ EOS
+ end
+
+ def down
+ execute 'DROP SCHEMA gitlab_partitions_dynamic'
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 0a77d788bab..3b6b7791201 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1,5 +1,9 @@
SET search_path=public;
+CREATE SCHEMA gitlab_partitions_dynamic;
+
+COMMENT ON SCHEMA gitlab_partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
+
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
CREATE TABLE public.abuse_reports (
@@ -14152,5 +14156,6 @@ COPY "schema_migrations" (version) FROM STDIN;
20200622235737
20200623000148
20200623000320
+20200623121135
\.
diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md
index efbd168c54e..8002a9ab296 100644
--- a/doc/administration/monitoring/performance/performance_bar.md
+++ b/doc/administration/monitoring/performance/performance_bar.md
@@ -31,7 +31,8 @@ From left to right, it displays:
![Redis profiling using the Performance Bar](img/performance_bar_redis_calls.png)
- **Elasticsearch calls**: the time taken (in milliseconds) and the total number of
Elasticsearch calls. Click to display a modal window with more details.
-- **Load timings** of the page: several values in milliseconds, separated by slashes.
+- **Load timings** of the page: if your browser supports load timings (Chromium
+ and Chrome) several values in milliseconds, separated by slashes.
Click to display a modal window with more details. The values, from left to right:
- **Backend**: time needed for the base page to load.
- [**First Contentful Paint**](https://web.dev/first-contentful-paint/):
diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md
index 3fa6ba40e6c..fd5dee69fc3 100644
--- a/doc/development/rake_tasks.md
+++ b/doc/development/rake_tasks.md
@@ -232,6 +232,9 @@ To see the full list of API routes, you can run:
bundle exec rake grape:path_helpers
```
+The generated list includes a full list of API endpoints and functional
+RESTful API verbs.
+
For the Rails controllers, run:
```shell
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 7e457c4982d..d4c1ce260e4 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -27,12 +27,18 @@ module Backup
progress.print "Dumping PostgreSQL database #{config['database']} ... "
pg_env
pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
+
if Gitlab.config.backup.pg_schema
- pgsql_args << "-n"
+ pgsql_args << '-n'
pgsql_args << Gitlab.config.backup.pg_schema
+
+ Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+ pgsql_args << '-n'
+ pgsql_args << schema.to_s
+ end
end
- spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
+ Process.spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
end
compress_wr.close
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index e7352a23b99..4d47a17545a 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -90,9 +90,7 @@ module Gitlab
end
def ordered_and_limited_query
- query
- .reorder(stage.end_event.timestamp_projection.desc)
- .limit(MAX_RECORDS)
+ order_by_end_event(query).limit(MAX_RECORDS)
end
def records
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
index 0ea98e82ecc..ed87d86f7e8 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -18,10 +18,14 @@ module Gitlab
end
def timestamp_projection
- Arel::Nodes::NamedFunction.new('COALESCE', [
+ Arel::Nodes::NamedFunction.new('COALESCE', column_list)
+ end
+
+ def column_list
+ [
issue_metrics_table[:first_associated_with_milestone_at],
issue_metrics_table[:first_added_to_board_at]
- ])
+ ]
end
# rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
index 4ca8745abe4..6f46b2a4180 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
@@ -10,6 +10,10 @@ module Gitlab
query.joins(:metrics)
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ def column_list
+ [timestamp_projection]
+ end
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index 37168a1fb0f..ab7fe0a6070 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -18,10 +18,14 @@ module Gitlab
end
def timestamp_projection
- Arel::Nodes::NamedFunction.new('COALESCE', [
+ Arel::Nodes::NamedFunction.new('COALESCE', column_list)
+ end
+
+ def column_list
+ [
issue_metrics_table[:first_associated_with_milestone_at],
issue_metrics_table[:first_added_to_board_at]
- ])
+ ]
end
# rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
index 619b45664fa..154fe9d3c38 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
@@ -21,6 +21,10 @@ module Gitlab
mr_metrics_table[:first_deployed_to_production_at]
end
+ def column_list
+ [timestamp_projection]
+ end
+
# rubocop: disable CodeReuse/ActiveRecord
def apply_query_customization(query)
query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
index 0c75a141c3c..8a8e8838ed8 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -32,6 +32,13 @@ module Gitlab
raise NotImplementedError
end
+ # List of columns that are referenced in the `timestamp_projection` expression
+ # Example timestamp projection: COALESCE(issue_metrics.created_at, issue_metrics.updated_at)
+ # Expected column list: issue_metrics.created_at, issue_metrics.updated_at
+ def column_list
+ []
+ end
+
# Optionally a StageEvent may apply additional filtering or join other tables on the base query.
def apply_query_customization(query)
query
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
index 29a2d55df1a..c9a75b39959 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -22,6 +22,29 @@ module Gitlab
stage.start_event.timestamp_projection
)
end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def order_by_end_event(query)
+ ordered_query = query.reorder(stage.end_event.timestamp_projection.desc)
+
+ # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause
+ if requires_grouping?
+ column_list = [
+ ordered_query.arel_table[:id],
+ *stage.end_event.column_list,
+ *stage.start_event.column_list
+ ]
+
+ ordered_query = ordered_query.group(column_list)
+ end
+
+ ordered_query
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def requires_grouping?
+ Array(params[:label_name]).size > 1
+ end
end
end
end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 02005be1f6a..8d28f09d2ad 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -22,6 +22,13 @@ module Gitlab
MIN_SCHEMA_VERSION = 20190506135400
MIN_SCHEMA_GITLAB_VERSION = '11.11.0'
+ # Schema we store dynamically managed partitions in
+ DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic
+
+ # This is an extensive list of postgres schemas owned by GitLab
+ # It does not include the default public schema
+ EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA].freeze
+
define_histogram :gitlab_database_transaction_seconds do
docstring "Time spent in database transactions, in seconds"
end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index abc4b2ba546..2acc51c1710 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -152,7 +152,7 @@ module Gitlab
end
def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
- if table_exists?(partition_name)
+ if table_exists?(table_for_range_partition(partition_name))
# rubocop:disable Gitlab/RailsLogger
Rails.logger.warn "Partition not created because it already exists" \
" (this may be due to an aborted migration or similar): partition_name: #{partition_name}"
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index ad10fd61067..34daafd06de 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -84,9 +84,13 @@ module Gitlab
private
+ def table_for_range_partition(partition_name)
+ "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition_name}"
+ end
+
def create_range_partition(partition_name, table_name, lower_bound, upper_bound)
execute(<<~SQL)
- CREATE TABLE #{partition_name} PARTITION OF #{table_name}
+ CREATE TABLE #{table_for_range_partition(partition_name)} PARTITION OF #{table_name}
FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound})
SQL
end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 4917d496d07..dc67c26042a 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -39,6 +39,11 @@ namespace :gitlab do
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
# Add `IF EXISTS` because cascade could have already deleted a table.
tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
+
+ # Drop all extra schema objects GitLab owns
+ Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+ connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
+ end
end
desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed'
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 98f9021cdc8..8150201d826 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -19057,6 +19057,9 @@ msgstr ""
msgid "Reporting"
msgstr ""
+msgid "Reports"
+msgstr ""
+
msgid "Reports|%{combinedString} and %{resolvedString}"
msgstr ""
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index 51ffd143836..be138992a60 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -202,6 +202,36 @@ RSpec.describe 'Database schema' do
end
end
+ context 'existence of Postgres schemas' do
+ def get_schemas
+ sql = <<~SQL
+ SELECT schema_name FROM
+ information_schema.schemata
+ WHERE
+ NOT schema_name ~* '^pg_' AND NOT schema_name = 'information_schema'
+ AND catalog_name = current_database()
+ SQL
+
+ ApplicationRecord.connection.select_all(sql).map do |row|
+ row['schema_name']
+ end
+ end
+
+ it 'we have a public schema' do
+ expect(get_schemas).to include('public')
+ end
+
+ Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+ it "we have a '#{schema}' schema'" do
+ expect(get_schemas).to include(schema.to_s)
+ end
+ end
+
+ it 'we do not have unexpected schemas' do
+ expect(get_schemas.size).to eq(Gitlab::Database::EXTRA_SCHEMAS.size + 1)
+ end
+ end
+
private
def retrieve_columns_name_with_jsonb
diff --git a/spec/helpers/ci_status_helper_spec.rb b/spec/helpers/ci/status_helper_spec.rb
index 1c8a749944b..12a6acb1ecc 100644
--- a/spec/helpers/ci_status_helper_spec.rb
+++ b/spec/helpers/ci/status_helper_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe CiStatusHelper do
+RSpec.describe Ci::StatusHelper do
include IconsHelper
let(:success_commit) { double("Ci::Pipeline", status: 'success') }
diff --git a/spec/lib/backup/database_spec.rb b/spec/lib/backup/database_spec.rb
new file mode 100644
index 00000000000..81a5219680f
--- /dev/null
+++ b/spec/lib/backup/database_spec.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Backup::Database do
+ let(:progress) { double('progress', print: nil, puts: nil) }
+
+ describe '#dump' do
+ subject { described_class.new(progress).dump }
+
+ let(:pg_schema) { nil }
+ let(:backup_config) { double('config', pg_schema: pg_schema, path: File.join(Rails.root, 'tmp')) }
+
+ before do
+ allow(Settings).to receive(:backup).and_return(backup_config)
+ allow(Process).to receive(:waitpid)
+ end
+
+ it 'does not limit pg_dump to any specific schema' do
+ expect(Process).to receive(:spawn) do |*cmd, _|
+ expect(cmd.join(' ')).not_to include('-n')
+ end
+
+ subject
+ end
+
+ it 'includes option to drop objects before restoration' do
+ expect(Process).to receive(:spawn) do |*cmd, _|
+ expect(cmd.join(' ')).to include('--clean')
+ end
+
+ subject
+ end
+
+ context 'with pg_schema configured explicitly' do
+ let(:pg_schema) { 'some_schema' }
+
+ it 'calls pg_dump' do
+ expect(Process).to receive(:spawn) do |*cmd, _|
+ expect(cmd.join(' ')).to start_with('pg_dump')
+ end
+
+ subject
+ end
+
+ it 'limits the psql dump to the specified schema' do
+ expect(Process).to receive(:spawn) do |*cmd, _|
+ expect(cmd.join(' ')).to include("-n #{pg_schema}")
+ end
+
+ subject
+ end
+
+ context 'extra schemas' do
+ Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+ it "includes the extra schema #{schema}" do
+ expect(Process).to receive(:spawn) do |*cmd, _|
+ expect(cmd.join(' ')).to include("-n #{schema}")
+ end
+
+ subject
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
index db7a65742c5..576ee1fc4c6 100644
--- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
+++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
@@ -275,7 +275,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
describe '#drop_partitioned_table_for' do
let(:expected_tables) do
- %w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
+ %w[000000 201912 202001 202002].map { |suffix| "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
end
context 'when the table is not allowed' do
diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb
index b47ce374c8e..7a54ce6b8b7 100644
--- a/spec/lib/gitlab/database_spec.rb
+++ b/spec/lib/gitlab/database_spec.rb
@@ -7,6 +7,14 @@ RSpec.describe Gitlab::Database do
stub_const('MigrationTest', Class.new { include Gitlab::Database })
end
+ describe 'EXTRA_SCHEMAS' do
+ it 'contains only schemas starting with gitlab_ prefix' do
+ described_class::EXTRA_SCHEMAS.each do |schema|
+ expect(schema.to_s).to start_with('gitlab_')
+ end
+ end
+ end
+
describe '.config' do
it 'returns a Hash' do
expect(described_class.config).to be_an_instance_of(Hash)
diff --git a/spec/support/helpers/partitioning_helpers.rb b/spec/support/helpers/partitioning_helpers.rb
index 98a13915d76..5d8466e1ef4 100644
--- a/spec/support/helpers/partitioning_helpers.rb
+++ b/spec/support/helpers/partitioning_helpers.rb
@@ -9,7 +9,7 @@ module PartitioningHelpers
end
def expect_range_partition_of(partition_name, table_name, min_value, max_value)
- definition = find_partition_definition(partition_name)
+ definition = find_partition_definition(partition_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
expect(definition).not_to be_nil
expect(definition['base_table']).to eq(table_name.to_s)
@@ -40,7 +40,7 @@ module PartitioningHelpers
SQL
end
- def find_partition_definition(partition)
+ def find_partition_definition(partition, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)
connection.select_one(<<~SQL)
select
parent_class.relname as base_table,
@@ -48,7 +48,10 @@ module PartitioningHelpers
from pg_class
inner join pg_inherits i on pg_class.oid = inhrelid
inner join pg_class parent_class on parent_class.oid = inhparent
- where pg_class.relname = '#{partition}' and pg_class.relispartition;
+ inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+ where pg_namespace.nspname = '#{schema}'
+ and pg_class.relname = '#{partition}'
+ and pg_class.relispartition
SQL
end
end
diff --git a/spec/support/shared_examples/lib/gitlab/cycle_analytics/event_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/cycle_analytics/event_shared_examples.rb
index a00359ce979..d0e41605e00 100644
--- a/spec/support/shared_examples/lib/gitlab/cycle_analytics/event_shared_examples.rb
+++ b/spec/support/shared_examples/lib/gitlab/cycle_analytics/event_shared_examples.rb
@@ -8,6 +8,7 @@ RSpec.shared_examples_for 'cycle analytics event' do
it { expect(described_class.identifier).to be_a_kind_of(Symbol) }
it { expect(instance.object_type.ancestors).to include(ApplicationRecord) }
it { expect(instance).to respond_to(:timestamp_projection) }
+ it { expect(instance.column_list).to be_a_kind_of(Array) }
describe '#apply_query_customization' do
it 'expects an ActiveRecord::Relation object as argument and returns a modified version of it' do