diff options
| -rw-r--r-- | db/migrate/20180420010616_cleanup_build_stage_migration.rb | 35 | ||||
| -rw-r--r-- | db/schema.rb | 1 | ||||
| -rw-r--r-- | doc/ci/yaml/README.md | 18 |
3 files changed, 46 insertions, 8 deletions
diff --git a/db/migrate/20180420010616_cleanup_build_stage_migration.rb b/db/migrate/20180420010616_cleanup_build_stage_migration.rb index 0342695ec3d..24777294101 100644 --- a/db/migrate/20180420010616_cleanup_build_stage_migration.rb +++ b/db/migrate/20180420010616_cleanup_build_stage_migration.rb @@ -2,6 +2,7 @@ class CleanupBuildStageMigration < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false + TMP_INDEX = 'tmp_id_stage_partial_null_index'.freeze disable_ddl_transaction! @@ -13,16 +14,48 @@ class CleanupBuildStageMigration < ActiveRecord::Migration end def up + disable_statement_timeout + + ## + # We steal from the background migrations queue to catch up with the + # scheduled migrations set. + # Gitlab::BackgroundMigration.steal('MigrateBuildStage') + ## + # We add temporary index, to make iteration over batches more performant. + # Conditional here is to avoid the need of doing that in a separate + # migration file to make this operation idempotent. + # + unless index_exists_by_name?(:ci_builds, TMP_INDEX) + add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', name: TMP_INDEX) + end + + ## + # We check if there are remaining rows that should be migrated (for example + # if Sidekiq / Redis fails / is restarted, what could result in not all + # background migrations being executed correctly. + # + # We migrate remaining rows synchronously in a blocking way, to make sure + # that when this migration is done we are confident that all rows are + # already migrated. + # Build.where('stage_id IS NULL').each_batch(of: 50) do |batch| range = batch.pluck('MIN(id)', 'MAX(id)').first Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range) end + + ## + # We remove temporary index, because it is not required during standard + # operations and runtime. + # + remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) end def down - # noop + if index_exists_by_name?(:ci_builds, TMP_INDEX) + remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) + end end end diff --git a/db/schema.rb b/db/schema.rb index 6fd10785d77..22a851f83a6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2148,6 +2148,7 @@ ActiveRecord::Schema.define(version: 20180508055821) do add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade add_foreign_key "ci_build_trace_sections", "projects", on_delete: :cascade add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify + add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index fb6d9826d08..2a17a51d7f8 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -738,10 +738,15 @@ cache: rspec: script: test cache: + key: rspec paths: - binaries/ ``` +Note that since cache is shared between jobs, if you're using different +paths for different jobs, you should also set a different **cache:key** +otherwise cache content can be overwritten. + ### `cache:key` > Introduced in GitLab Runner v1.0.0. @@ -756,10 +761,9 @@ or any other way that fits your workflow. This way, you can fine tune caching, allowing you to cache data between different jobs or even different branches. The `cache:key` variable can use any of the -[predefined variables](../variables/README.md), and the default key, if not set, -is `$CI_JOB_NAME-$CI_COMMIT_REF_NAME` which translates as "per-job and -per-branch". It is the default across the project, therefore everything is -shared between pipelines and jobs running on the same branch by default. +[predefined variables](../variables/README.md), and the default key, if not +set, is just literal `default` which means everything is shared between each +pipelines and jobs by default, starting from GitLab 9.0. NOTE: **Note:** The `cache:key` variable cannot contain the `/` character, or the equivalent @@ -779,7 +783,7 @@ If you use **Windows Batch** to run your shell scripts you need to replace ```yaml cache: - key: "%CI_JOB_STAGE%-%CI_COMMIT_REF_SLUG%" + key: "%CI_COMMIT_REF_SLUG%" paths: - binaries/ ``` @@ -789,7 +793,7 @@ If you use **Windows PowerShell** to run your shell scripts you need to replace ```yaml cache: - key: "$env:CI_JOB_STAGE-$env:CI_COMMIT_REF_SLUG" + key: "$env:CI_COMMIT_REF_SLUG" paths: - binaries/ ``` @@ -1572,7 +1576,7 @@ capitalization, the commit will be created but the pipeline will be skipped. ## Validate the .gitlab-ci.yml Each instance of GitLab CI has an embedded debug tool called Lint, which validates the -content of your `.gitlab-ci.yml` files. You can find the Lint under the page `ci/lint` of your +content of your `.gitlab-ci.yml` files. You can find the Lint under the page `ci/lint` of your project namespace (e.g, `http://gitlab-example.com/gitlab-org/project-123/-/ci/lint`) ## Using reserved keywords |