diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-10 00:09:22 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-10 00:09:22 +0000 |
commit | cd5b3f1cc443b7c46ddd7ea43ce0a14889cd128b (patch) | |
tree | 915594db59f3cfc52470cc1cbe8e979324db88e7 | |
parent | 94a2edbe7984384df4c3fb7efa308af2e0f42d8c (diff) | |
download | gitlab-ce-cd5b3f1cc443b7c46ddd7ea43ce0a14889cd128b.tar.gz |
Add latest changes from gitlab-org/gitlab@master
25 files changed, 391 insertions, 217 deletions
diff --git a/app/assets/javascripts/ci/runner/components/cells/runner_summary_cell.vue b/app/assets/javascripts/ci/runner/components/cells/runner_summary_cell.vue index 77117efa16a..d5bc39ff039 100644 --- a/app/assets/javascripts/ci/runner/components/cells/runner_summary_cell.vue +++ b/app/assets/javascripts/ci/runner/components/cells/runner_summary_cell.vue @@ -86,7 +86,7 @@ export default { > {{ runner.description }} </tooltip-on-truncate> - <span v-else class="gl-text-gray-500">{{ $options.i18n.I18N_NO_DESCRIPTION }}</span> + <span v-else class="gl-text-secondary">{{ $options.i18n.I18N_NO_DESCRIPTION }}</span> </div> <div> diff --git a/app/assets/javascripts/ci/runner/components/runner_detail.vue b/app/assets/javascripts/ci/runner/components/runner_detail.vue index c260670b517..9e8055a8432 100644 --- a/app/assets/javascripts/ci/runner/components/runner_detail.vue +++ b/app/assets/javascripts/ci/runner/components/runner_detail.vue @@ -49,7 +49,7 @@ export default { <template v-if="value || $scopedSlots.value"> <slot name="value">{{ value }}</slot> </template> - <span v-else class="gl-text-gray-500">{{ emptyValue }}</span> + <span v-else class="gl-text-secondary">{{ emptyValue }}</span> </dd> </div> </template> diff --git a/app/assets/javascripts/ci/runner/components/runner_groups.vue b/app/assets/javascripts/ci/runner/components/runner_groups.vue index c3b35bd52a9..8501d165157 100644 --- a/app/assets/javascripts/ci/runner/components/runner_groups.vue +++ b/app/assets/javascripts/ci/runner/components/runner_groups.vue @@ -32,6 +32,6 @@ export default { :avatar-url="group.avatarUrl" /> </template> - <span v-else class="gl-text-gray-500">{{ __('None') }}</span> + <span v-else class="gl-text-secondary">{{ __('None') }}</span> </div> </template> diff --git a/app/assets/javascripts/ci/runner/components/runner_projects.vue b/app/assets/javascripts/ci/runner/components/runner_projects.vue index 84008e8eee8..4a6e90b44a9 100644 --- a/app/assets/javascripts/ci/runner/components/runner_projects.vue +++ b/app/assets/javascripts/ci/runner/components/runner_projects.vue @@ -133,7 +133,7 @@ export default { :is-owner="isOwner(project.id)" /> </template> - <div v-else class="gl-py-5 gl-text-gray-500">{{ $options.I18N_NO_PROJECTS_FOUND }}</div> + <div v-else class="gl-py-5 gl-text-secondary">{{ $options.I18N_NO_PROJECTS_FOUND }}</div> <runner-pagination :disabled="loading" diff --git a/app/controllers/omniauth_callbacks_controller.rb b/app/controllers/omniauth_callbacks_controller.rb index 0d6f289f5f5..8650b6cbc6f 100644 --- a/app/controllers/omniauth_callbacks_controller.rb +++ b/app/controllers/omniauth_callbacks_controller.rb @@ -119,6 +119,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController identity_linker ||= auth_module::IdentityLinker.new(current_user, oauth, session) link_identity(identity_linker) + set_remember_me(current_user) if identity_linker.changed? redirect_identity_linked diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index f53e1c72dcb..039ad7c5f03 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -39,6 +39,7 @@ exceptions: - CORE - CORS - CPU + - CRAN - CRIME - CRM - CRUD @@ -188,6 +189,7 @@ exceptions: - SAST - SATA - SBOM + - SBT - SCIM - SCM - SCP diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index fa702cedc6f..81b21f026f4 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -196,6 +196,7 @@ corpuses Coursier CPU CPUs +CRAN cron crond cronjob @@ -826,6 +827,7 @@ Salesforce sandboxing sanitization SBOMs +SBT sbt scalers scatterplot diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 8b7f49d77d9..b304aaa3adc 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -482,7 +482,7 @@ To migrate existing local data to object storage see the following guides: - [LFS objects](lfs/index.md#migrating-to-object-storage) - [Uploads](raketasks/uploads/migrate.md#migrate-to-object-storage) - [Merge request diffs](merge_request_diffs.md#using-object-storage) -- [Packages](packages/index.md#migrating-local-packages-to-object-storage) (optional feature) +- [Packages](packages/index.md#migrate-local-packages-to-object-storage) (optional feature) - [Dependency Proxy](packages/dependency_proxy.md#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage) - [Terraform state files](terraform_state.md#migrate-to-object-storage) - [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage) @@ -537,7 +537,7 @@ supported by consolidated configuration form, refer to the following guides: | [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | | [Merge request diffs](merge_request_diffs.md#using-object-storage) | **{check-circle}** Yes | | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| **{dotted-circle}** No | -| [Packages](packages/index.md#using-object-storage) (optional feature) | **{check-circle}** Yes | +| [Packages](packages/index.md#use-object-storage) (optional feature) | **{check-circle}** Yes | | [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) | **{check-circle}** Yes | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | | [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | @@ -648,10 +648,6 @@ if this access is not in place include: See [the LFS documentation](lfs/index.md#error-viewing-a-pdf-file) for more details. -Getting a `403 Forbidden` response is specifically called out on the -[package repository documentation](packages/index.md#using-object-storage) -as a side effect of how some build tools work. - Additionally for a short time period users could share pre-signed, time-limited object storage URLs with others without authentication. Also bandwidth charges may be incurred between the object storage provider and the client. diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 74d835eb744..6e53d77109f 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -6,11 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Package Registry administration **(FREE SELF)** -GitLab Packages allows organizations to use GitLab as a private repository -for a variety of common package managers. Users are able to build and publish -packages, which can be easily consumed as a dependency in downstream projects. +To use GitLab as a private repository for a variety of common package managers, use the Package Registry. +You can build and publish +packages, which can be consumed as dependencies in downstream projects. -The Packages feature allows GitLab to act as a repository and supports the following formats: +## Supported formats + +The Package Registry supports the following formats: | Package type | GitLab version | |-------------------------------------------------------------------|----------------| @@ -49,204 +51,155 @@ guides you through the process. <!-- vale gitlab.Spelling = YES --> -## Enabling the Packages feature +## Rate limits -NOTE: -After the Packages feature is enabled, the repositories are available -for all new projects by default. To enable it for existing projects, users -explicitly do so in the project's settings. +When downloading packages as dependencies in downstream projects, many requests are made through the +Packages API. You may therefore reach enforced user and IP rate limits. To address this issue, you +can define specific rate limits for the Packages API. For more details, see [Package Registry Rate Limits](../../user/admin_area/settings/package_registry_rate_limits.md). -To enable the Packages feature: +## Change the storage path -**Omnibus GitLab installations** +By default, the packages are stored locally, but you can change the default +local location or even use object storage. -1. Edit `/etc/gitlab/gitlab.rb` and add the following line: +### Change the local storage path - ```ruby - gitlab_rails['packages_enabled'] = true - ``` +By default, the packages are stored in a local path, relative to the GitLab +installation: -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +- Linux package (Omnibus): `/var/opt/gitlab/gitlab-rails/shared/packages/` +- Self-compiled (source): `/home/git/gitlab/shared/packages/` -**Installations from source** +To change the local storage path: -1. After the installation is complete, you configure the `packages` - section in `config/gitlab.yml`. Set to `true` to enable it: +::Tabs - ```yaml - packages: - enabled: true +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb` and add the following line: + + ```ruby + gitlab_rails['packages_storage_path'] = "/mnt/packages" ``` -1. [Restart GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` -**Helm Chart installations** +:::TabTitle Self-compiled (source) -1. After the installation is complete, you configure the `packages` - section in `global.appConfig.packages`. Set to `true` to enable it: +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - packages: - enabled: true + production: &base + packages: + enabled: true + storage_path: /mnt/packages ``` -1. [Restart GitLab](../restart_gitlab.md#helm-chart-installations) for the changes to take effect. +1. Save the file and restart GitLab: -## Rate limits + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target -When downloading packages as dependencies in downstream projects, many requests are made through the -Packages API. You may therefore reach enforced user and IP rate limits. To address this issue, you -can define specific rate limits for the Packages API. For more details, see [Package Registry Rate Limits](../../user/admin_area/settings/package_registry_rate_limits.md). + # For systems running SysV init + sudo service gitlab restart + ``` -## Changing the storage path +::EndTabs -By default, the packages are stored locally, but you can change the default -local location or even use object storage. +Docker and Kubernetes do not use local storage. -### Changing the local storage path +- For the Helm chart (Kubernetes): Use object storage instead. +- For Docker: The `/var/opt/gitlab/` directory is already + mounted in a directory on the host. There's no need to change the local + storage path inside the container. -The packages for Omnibus GitLab installations are stored under -`/var/opt/gitlab/gitlab-rails/shared/packages/` and for source -installations under `shared/packages/` (relative to the Git home directory). -To change the local storage path: +### Use object storage -**Omnibus GitLab installations** +Instead of relying on the local storage, you can use an object storage to store +packages. -1. Edit `/etc/gitlab/gitlab.rb` and add the following line: +For more information, see how to use the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). - ```ruby - gitlab_rails['packages_storage_path'] = "/mnt/packages" - ``` +### Migrate local packages to object storage + +After [configuring the object storage](#use-object-storage), use the following task to +migrate existing packages from the local storage to the remote storage. +The processing is done in a background worker and requires **no downtime**. -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. +1. Migrate the packages. -**Installations from source** + ::Tabs -1. Edit the `packages` section in `config/gitlab.yml`: + :::TabTitle Linux package (Omnibus) - ```yaml - packages: - enabled: true - storage_path: shared/packages + ```shell + sudo gitlab-rake "gitlab:packages:migrate" ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -### Using object storage + :::TabTitle Self-compiled (source) -Instead of relying on the local storage, you can use an object storage to -store packages. + ```shell + RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:packages:migrate + ``` -[Read more about using object storage with GitLab](../object_storage.md). + ::EndTabs -NOTE: -We recommend using the [consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original configuration format. +1. Track the progress and verify that all packages migrated successfully using + the PostgreSQL console. -**Omnibus GitLab installations** + ::Tabs -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where - necessary): + :::TabTitle Linux package (Omnibus) 14.1 and earlier - ```ruby - gitlab_rails['packages_enabled'] = true - gitlab_rails['packages_object_store_enabled'] = true - gitlab_rails['packages_object_store_remote_directory'] = "packages" # The bucket name. - gitlab_rails['packages_object_store_proxy_download'] = false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. - gitlab_rails['packages_object_store_connection'] = { - ## - ## If the provider is AWS S3, uncomment the following - ## - #'provider' => 'AWS', - #'region' => 'eu-west-1', - #'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', - #'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY', - ## If an IAM profile is being used with AWS, omit the aws_access_key_id and aws_secret_access_key and uncomment - #'use_iam_profile' => true, - ## - ## If the provider is other than AWS (an S3-compatible one), uncomment the following - ## - #'host' => 's3.amazonaws.com', - #'aws_signature_version' => 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - #'endpoint' => 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - #'path_style' => false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. - } + ```shell + sudo gitlab-rails dbconsole ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - -**Installations from source** + :::TabTitle Linux package (Omnibus) 14.2 and later -1. Edit the `packages` section in `config/gitlab.yml` (uncomment where necessary): - - ```yaml - packages: - enabled: true - ## - ## The location where build packages are stored (default: shared/packages). - ## - # storage_path: shared/packages - object_store: - enabled: false - remote_directory: packages # The bucket name. - # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. - connection: - ## - ## If the provider is AWS S3, use the following: - ## - provider: AWS - region: us-east-1 - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - ## - ## If the provider is other than AWS (an S3-compatible one), comment out the previous 4 lines and use the following instead: - ## - # host: 's3.amazonaws.com' # default: s3.amazonaws.com. - # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - # endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - # path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. + ```shell + sudo gitlab-rails dbconsole --database main ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -### Migrating local packages to object storage + :::TabTitle Self-compiled (source) -After [configuring the object storage](#using-object-storage), use the following task to -migrate existing packages from the local storage to the remote storage. -The processing is done in a background worker and requires **no downtime**. + ```shell + RAILS_ENV=production sudo -u git -H psql -d gitlabhq_production + ``` -For Omnibus GitLab: + ::EndTabs -```shell -sudo gitlab-rake "gitlab:packages:migrate" -``` +1. Verify that all packages migrated to object storage with the following SQL + query. The number of `objectstg` should be the same as `total`: -For installations from source: + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM packages_package_files; -```shell -RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:packages:migrate -``` + total | filesystem | objectstg + ------+------------+----------- + 34 | 0 | 34 + ``` -You can optionally track progress and verify that all packages migrated successfully using the -[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database): +1. Finally, verify that there are no files on disk in the `packages` directory: -- `sudo gitlab-rails dbconsole` for Omnibus GitLab 14.1 and earlier. -- `sudo gitlab-rails dbconsole --database main` for Omnibus GitLab 14.2 and later. -- `sudo -u git -H psql -d gitlabhq_production` for source-installed instances. + ::Tabs -Verify `objectstg` below (where `file_store = '2'`) has count of all packages: + :::TabTitle Linux package (Omnibus) -```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM packages_package_files; + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/packages -type f | grep -v tmp | wc -l + ``` -total | filesystem | objectstg -------+------------+----------- - 34 | 0 | 34 -``` + :::TabTitle Self-compiled (source) -Verify that there are no files on disk in the `packages` folder: + ```shell + sudo -u git find /home/git/gitlab/shared/packages -type f | grep -v tmp | wc -l + ``` -```shell -sudo find /var/opt/gitlab/gitlab-rails/shared/packages -type f | grep -v tmp | wc -l -``` + ::EndTabs diff --git a/doc/integration/glab/img/example-output_v15_6.png b/doc/integration/glab/img/example-output_v15_6.png Binary files differdeleted file mode 100644 index 7d2c5f2e4d9..00000000000 --- a/doc/integration/glab/img/example-output_v15_6.png +++ /dev/null diff --git a/doc/integration/glab/img/glabgettingstarted.gif b/doc/integration/glab/img/glabgettingstarted.gif Binary files differnew file mode 100644 index 00000000000..cf335294e41 --- /dev/null +++ b/doc/integration/glab/img/glabgettingstarted.gif diff --git a/doc/integration/glab/index.md b/doc/integration/glab/index.md index 77239c5f6c5..3951f38dfab 100644 --- a/doc/integration/glab/index.md +++ b/doc/integration/glab/index.md @@ -14,7 +14,7 @@ switching between windows and browser tabs. - Work with merge requests. - Watch running pipelines directly from your CLI. -![command example](img/example-output_v15_6.png) +![command example](img/glabgettingstarted.gif) The GitLab CLI uses commands structured like `glab <command> <subcommand> [flags]` to perform many of the actions you normally do from the GitLab user interface: diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 15593655fac..a60e80d6196 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -342,6 +342,7 @@ To link SAML to your existing GitLab.com account: 1. Sign in to your GitLab.com account. [Reset your password](https://gitlab.com/users/password/new) if necessary. 1. Locate and visit the **GitLab single sign-on URL** for the group you're signing in to. A group owner can find this on the group's **Settings > SAML SSO** page. If the sign-in URL is configured, users can connect to the GitLab app from the identity provider. +1. Optional. Select the **Remember me** checkbox to stay signed in to GitLab for 2 weeks. You may still be asked to re-authenticate with your SAML provider more frequently. 1. Select **Authorize**. 1. Enter your credentials on the identity provider if prompted. 1. You are then redirected back to GitLab.com and should now have access to the group. In the future, you can use SAML to sign in to GitLab.com. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index b1b5e99707b..c62999100c1 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -332,7 +332,7 @@ This might be accompanied by another error: This is usually a permissions issue with either: - `'packages_storage_path'` default `/var/opt/gitlab/gitlab-rails/shared/packages/`. -- The remote bucket if [object storage](../../../administration/packages/index.md#using-object-storage) +- The remote bucket if [object storage](../../../administration/packages/index.md#use-object-storage) is used. In the latter case, ensure the bucket exists and GitLab has write access to it. diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 64401bc0674..973ab20f547 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -9,55 +9,68 @@ module Gitlab # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + include Gitlab::ClassAttributes - def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: - ) + DEFAULT_FEATURE_CATEGORY = :database - @start_id = start_id - @end_id = end_id - @batch_table = batch_table - @batch_column = batch_column - @sub_batch_size = sub_batch_size - @pause_ms = pause_ms - @job_arguments = job_arguments - @connection = connection - end + class << self + def generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) + new( + batch_table: batch_table, batch_column: batch_column, + job_arguments: job_arguments, connection: connection, + start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 + ) + end - def self.generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) - new( - batch_table: batch_table, batch_column: batch_column, - job_arguments: job_arguments, connection: connection, - start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 - ) - end + def job_arguments_count + 0 + end - def self.job_arguments_count - 0 - end + def operation_name(operation) + define_method('operation_name') do + operation + end + end - def self.operation_name(operation) - define_method('operation_name') do - operation + def job_arguments(*args) + args.each.with_index do |arg, index| + define_method(arg) do + @job_arguments[index] + end + end + + define_singleton_method(:job_arguments_count) do + args.count + end end - end - def self.job_arguments(*args) - args.each.with_index do |arg, index| - define_method(arg) do - @job_arguments[index] + def scope_to(scope) + define_method(:filter_batch) do |relation| + instance_exec(relation, &scope) end end - define_singleton_method(:job_arguments_count) do - args.count + def feature_category(feature_category_name = nil) + if feature_category_name.present? + set_class_attribute(:feature_category, feature_category_name) + else + get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY + end end end - def self.scope_to(scope) - define_method(:filter_batch) do |relation| - instance_exec(relation, &scope) - end + def initialize( + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + ) + + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @job_arguments = job_arguments + @connection = connection end def filter_batch(relation) diff --git a/lib/gitlab/database/migrations/batched_migration_last_id.rb b/lib/gitlab/database/migrations/batched_migration_last_id.rb new file mode 100644 index 00000000000..c77a2e9a375 --- /dev/null +++ b/lib/gitlab/database/migrations/batched_migration_last_id.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BatchedMigrationLastId + FILE_NAME = 'last-batched-background-migration-id.txt' + + def initialize(connection, base_dir) + @connection = connection + @base_dir = base_dir + end + + def store + File.open(file_path, 'wb') { |file| file.write(last_background_migration_id) } + end + + # Reads the last id from the file + # + # @info casts the file content into an +Integer+. + # Casts any unexpected content to +nil+ + # + # @example + # Integer('4', exception: false) # => 4 + # Integer('', exception: false) # => nil + # + # @return [Integer, nil] + def read + return unless File.exist?(file_path) + + Integer(File.read(file_path).presence, exception: false) + end + + private + + attr_reader :connection, :base_dir + + def file_path + @file_path ||= base_dir.join(FILE_NAME) + end + + def last_background_migration_id + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 27b161419b2..ed55081c9ab 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -29,16 +29,14 @@ module Gitlab def batched_background_migrations(for_database:, legacy_mode: false) runner = nil - result_dir = if legacy_mode - BASE_RESULT_DIR.join('background_migrations') - else - BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') - end + result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + from_id = batched_migrations_last_id(for_database).read + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: result_dir, connection: connection) + .new(result_dir: result_dir, connection: connection, from_id: from_id) end runner @@ -66,6 +64,18 @@ module Gitlab end # rubocop:enable Database/MultipleDatabases + def batched_migrations_last_id(for_database) + runner = nil + base_dir = background_migrations_dir(for_database, false) + + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::BatchedMigrationLastId + .new(connection, base_dir) + end + + runner + end + private def migrations_for_up(database) @@ -90,6 +100,12 @@ module Gitlab existing_versions.include?(migration.version) && versions_this_branch.include?(migration.version) end end + + def background_migrations_dir(db, legacy_mode) + return BASE_RESULT_DIR.join('background_migrations') if legacy_mode + + BASE_RESULT_DIR.join(db.to_s, 'background_migrations') + end end attr_reader :direction, :result_dir, :migrations diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 46855ca1921..a16103f452c 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,16 +6,17 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - def initialize(result_dir:, connection:) + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection + @from_id = from_id end def jobs_by_migration_name Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .where('id > ?', from_id) .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) @@ -102,6 +103,10 @@ module Gitlab end end end + + private + + attr_reader :from_id end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4ef0c396f4a..95e290c02e0 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -316,6 +316,7 @@ namespace :gitlab do all_databases.each do |db| desc "Run migrations on #{db} with instrumentation" task db => :environment do + Gitlab::Database::Migrations::Runner.batched_migrations_last_id(db).store Gitlab::Database::Migrations::Runner.up(database: db).run end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index a48b08ea0f1..113f4be743c 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -35980,13 +35980,16 @@ msgstr "" msgid "SAML single sign-on for %{group_name}" msgstr "" +msgid "SAML|Sign in to %{groupName}" +msgstr "" + msgid "SAML|Sign in to GitLab to connect your organization's account" msgstr "" msgid "SAML|The %{strongOpen}%{group_path}%{strongClose} group allows you to sign in using single sign-on." msgstr "" -msgid "SAML|To access %{strongOpen}%{group_name}%{strongClose}, you must sign in using single sign-on through an external sign-in page." +msgid "SAML|To access %{groupName}, you must sign in using single sign-on through an external sign-in page." msgstr "" msgid "SAML|To allow %{strongOpen}%{group_name}%{strongClose} to manage your GitLab account %{strongOpen}%{username}%{strongClose} (%{email}) after you sign in successfully using single sign-on, select %{strongOpen}Authorize%{strongClose}." @@ -36184,12 +36187,18 @@ msgstr "" msgid "ScanResultPolicy|add an approver" msgstr "" +msgid "ScanResultPolicy|from %{scanners} find(s) more than %{vulnerabilitiesAllowed} %{severities} %{vulnerabilityStates} vulnerabilities in an open merge request targeting %{branches}" +msgstr "" + msgid "ScanResultPolicy|scanners" msgstr "" msgid "ScanResultPolicy|severity levels" msgstr "" +msgid "ScanResultPolicy|vulnerabilities allowed" +msgstr "" + msgid "ScanResultPolicy|vulnerability states" msgstr "" @@ -37121,6 +37130,9 @@ msgstr "" msgid "SecurityOrchestration|branches" msgstr "" +msgid "SecurityOrchestration|group level branch" +msgstr "" + msgid "SecurityOrchestration|scanner finds" msgstr "" @@ -38448,9 +38460,6 @@ msgstr "" msgid "Sign in preview" msgstr "" -msgid "Sign in to %{group_name}" -msgstr "" - msgid "Sign in to GitLab" msgstr "" @@ -40247,6 +40256,9 @@ msgstr "" msgid "SuperSonics|You can start a free trial of GitLab Ultimate without any obligation or payment details." msgstr "" +msgid "SuperSonics|You can sync your subscription data to ensure your details are up to date." +msgstr "" + msgid "SuperSonics|You do not have an active subscription" msgstr "" diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index 95be14cefb1..7280ca0b58e 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -158,6 +158,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do end end + describe '.feature_category' do + context 'when jobs does not have feature_category attribute set' do + let(:job_class) { Class.new(described_class) } + + it 'returns :database as default' do + expect(job_class.feature_category).to eq(:database) + end + end + + context 'when jobs have feature_category attribute set' do + let(:job_class) do + Class.new(described_class) do + feature_category :delivery + end + end + + it 'returns the provided value' do + expect(job_class.feature_category).to eq(:delivery) + end + end + end + describe 'descendants', :eager_load do it 'have the same method signature for #perform' do expected_arity = described_class.instance_method(:perform).arity diff --git a/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb b/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb new file mode 100644 index 00000000000..97b432406eb --- /dev/null +++ b/spec/lib/gitlab/database/migrations/batched_migration_last_id_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Migrations::BatchedMigrationLastId, feature_category: :pipeline_insights do + subject(:test_sampling) { described_class.new(connection, base_dir) } + + let(:base_dir) { Pathname.new(Dir.mktmpdir) } + let(:file_name) { 'last-batched-background-migration-id.txt' } + let(:file_path) { base_dir.join(file_name) } + let(:file_contents) { nil } + + where(:base_model) do + [ + [ApplicationRecord], [Ci::ApplicationRecord] + ] + end + + with_them do + let(:connection) { base_model.connection } + + after do + FileUtils.rm_rf(file_path) + end + + describe '#read' do + before do + File.write(file_path, file_contents) + end + + context 'when the file exists and have content' do + let(:file_contents) { 99 } + + it { expect(test_sampling.read).to eq(file_contents) } + end + + context 'when the file exists and is blank' do + it { expect(test_sampling.read).to be_nil } + end + + context "when the file doesn't exists" do + before do + FileUtils.rm_rf(file_path) + end + + it { expect(test_sampling.read).to be_nil } + end + end + + describe '#store' do + let(:file_contents) { File.read(file_path) } + let(:migration) do + Gitlab::Database::SharedModel.using_connection(connection) do + create(:batched_background_migration) + end + end + + it 'creates the file properly' do + test_sampling.store + + expect(File).to exist(file_path) + end + + it 'stores the proper id in the file' do + migration + test_sampling.store + + expect(file_contents).to eq(migration.id.to_s) + end + end + end +end diff --git a/spec/lib/gitlab/database/migrations/runner_spec.rb b/spec/lib/gitlab/database/migrations/runner_spec.rb index bd382547689..66eb5a5de51 100644 --- a/spec/lib/gitlab/database/migrations/runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/runner_spec.rb @@ -230,5 +230,13 @@ RSpec.describe Gitlab::Database::Migrations::Runner, :reestablished_active_recor end end end + + describe '.batched_migrations_last_id' do + let(:runner_class) { Gitlab::Database::Migrations::BatchedMigrationLastId } + + it 'matches the expected runner class' do + expect(described_class.batched_migrations_last_id(database)).to be_a(runner_class) + end + end end end diff --git a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb index 07226f3d025..73d69d55e5a 100644 --- a/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb +++ b/spec/lib/gitlab/database/migrations/test_batched_background_runner_spec.rb @@ -55,6 +55,8 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez let(:table_name) { "_test_column_copying" } + let(:from_id) { 0 } + before do connection.execute(<<~SQL) CREATE TABLE #{table_name} ( @@ -76,7 +78,8 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez end subject(:sample_migration) do - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 1.minute) + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 1.minute) end it 'runs sampled jobs from the batched background migration' do @@ -111,20 +114,26 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez job_interval: 5.minutes, batch_size: 100) - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 3.minutes) + described_class.new(result_dir: result_dir, connection: connection, + from_id: from_id).run_jobs(for_duration: 3.minutes) expect(calls).not_to be_empty end context 'with multiple jobs to run' do - it 'runs all jobs created within the last 3 hours' do + let(:last_id) do + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + + it 'runs all pending jobs based on the last migration id' do old_migration = define_background_migration(migration_name) queue_migration(migration_name, table_name, :id, job_interval: 5.minutes, batch_size: 100) - travel 4.hours - + last_id new_migration = define_background_migration('NewMigration') { travel 1.second } queue_migration('NewMigration', table_name, :id, job_interval: 5.minutes, @@ -138,14 +147,15 @@ RSpec.describe Gitlab::Database::Migrations::TestBatchedBackgroundRunner, :freez sub_batch_size: 5) expect_migration_runs(new_migration => 3, other_new_migration => 2, old_migration => 0) do - described_class.new(result_dir: result_dir, connection: connection).run_jobs(for_duration: 5.seconds) + described_class.new(result_dir: result_dir, connection: connection, + from_id: last_id).run_jobs(for_duration: 5.seconds) end end end end context 'choosing uniform batches to run' do - subject { described_class.new(result_dir: result_dir, connection: connection) } + subject { described_class.new(result_dir: result_dir, connection: connection, from_id: from_id) } describe '#uniform_fractions' do it 'generates evenly distributed sequences of fractions' do diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 08bec9fda78..22abfc33d1b 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -701,6 +701,16 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do describe '#up' do subject { run_rake_task("gitlab:db:migration_testing:up:#{db}") } + let(:migrations_id_runner) do + instance_double('Gitlab::Database::Migrations::BatchedMigrationLastId', store: true) + end + + before do + allow(::Gitlab::Database::Migrations::Runner).to( + receive(:batched_migrations_last_id).and_return(migrations_id_runner) + ) + end + it 'delegates to the migration runner' do expect(::Gitlab::Database::Migrations::Runner).to receive(:up).with(database: db).and_return(runner) expect(runner).to receive(:run) |