diff options
40 files changed, 697 insertions, 51 deletions
diff --git a/app/assets/javascripts/behaviors/requires_input.js b/app/assets/javascripts/behaviors/requires_input.js index c09d9ccddd6..d8056e48d4e 100644 --- a/app/assets/javascripts/behaviors/requires_input.js +++ b/app/assets/javascripts/behaviors/requires_input.js @@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) { } $(() => { - const $form = $('form.js-requires-input'); - if ($form) { + $('form.js-requires-input').each((i, el) => { + const $form = $(el); + $form.requiresInput(); hideOrShowHelpBlock($form); $('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form)); - } + }); }); diff --git a/app/assets/javascripts/lib/utils/file_upload.js b/app/assets/javascripts/lib/utils/file_upload.js new file mode 100644 index 00000000000..b41ffb44971 --- /dev/null +++ b/app/assets/javascripts/lib/utils/file_upload.js @@ -0,0 +1,13 @@ +export default (buttonSelector, fileSelector) => { + const btn = document.querySelector(buttonSelector); + const fileInput = document.querySelector(fileSelector); + const form = btn.closest('form'); + + btn.addEventListener('click', () => { + fileInput.click(); + }); + + fileInput.addEventListener('change', () => { + form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape + }); +}; diff --git a/app/assets/javascripts/pages/projects/edit/index.js b/app/assets/javascripts/pages/projects/edit/index.js index f5b1cf85e68..899d5925956 100644 --- a/app/assets/javascripts/pages/projects/edit/index.js +++ b/app/assets/javascripts/pages/projects/edit/index.js @@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels'; import setupProjectEdit from '~/project_edit'; import initConfirmDangerModal from '~/confirm_danger_modal'; import mountBadgeSettings from '~/pages/shared/mount_badge_settings'; +import fileUpload from '~/lib/utils/file_upload'; import initProjectLoadingSpinner from '../shared/save_project_loader'; -import projectAvatar from '../shared/project_avatar'; import initProjectPermissionsSettings from '../shared/permissions'; document.addEventListener('DOMContentLoaded', () => { @@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => { setupProjectEdit(); // Initialize expandable settings panels initSettingsPanels(); - projectAvatar(); + fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input'); initProjectPermissionsSettings(); initConfirmDangerModal(); mountBadgeSettings(PROJECT_BADGE); diff --git a/app/assets/javascripts/pages/projects/settings/repository/form.js b/app/assets/javascripts/pages/projects/settings/repository/form.js index a52861c9efa..3e02893f24c 100644 --- a/app/assets/javascripts/pages/projects/settings/repository/form.js +++ b/app/assets/javascripts/pages/projects/settings/repository/form.js @@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys'; import ProtectedBranchCreate from '~/protected_branches/protected_branch_create'; import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list'; import DueDateSelectors from '~/due_date_select'; +import fileUpload from '~/lib/utils/file_upload'; export default () => { new ProtectedTagCreate(); @@ -16,4 +17,5 @@ export default () => { new ProtectedBranchCreate(); new ProtectedBranchEditList(); new DueDateSelectors(); + fileUpload('.js-choose-file', '.js-object-map-input'); }; diff --git a/app/assets/javascripts/pages/projects/shared/project_avatar.js b/app/assets/javascripts/pages/projects/shared/project_avatar.js deleted file mode 100644 index 1e69ecb481d..00000000000 --- a/app/assets/javascripts/pages/projects/shared/project_avatar.js +++ /dev/null @@ -1,16 +0,0 @@ -import $ from 'jquery'; - -export default function projectAvatar() { - $('.js-choose-project-avatar-button').bind('click', function onClickAvatar() { - const form = $(this).closest('form'); - return form.find('.js-project-avatar-input').click(); - }); - - $('.js-project-avatar-input').bind('change', function onClickAvatarInput() { - const form = $(this).closest('form'); - const filename = $(this) - .val() - .replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape - return form.find('.js-avatar-filename').text(filename); - }); -} diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index 1d76c90d4eb..30724de7f6a 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -5,6 +5,7 @@ module Projects class RepositoryController < Projects::ApplicationController before_action :authorize_admin_project! before_action :remote_mirror, only: [:show] + before_action :check_cleanup_feature_flag!, only: :cleanup def show render_show @@ -20,8 +21,26 @@ module Projects render_show end + def cleanup + cleanup_params = params.require(:project).permit(:bfg_object_map) + result = Projects::UpdateService.new(project, current_user, cleanup_params).execute + + if result[:status] == :success + RepositoryCleanupWorker.perform_async(project.id, current_user.id) + flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.') + else + flash[:alert] = _('Failed to upload object map file') + end + + redirect_to project_settings_repository_path(project) + end + private + def check_cleanup_feature_flag! + render_404 unless ::Feature.enabled?(:project_cleanup, project) + end + def render_show @deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user) @deploy_tokens = @project.deploy_tokens.active diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 0a7f930110a..fcec6d2c9d3 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -257,6 +257,10 @@ module ProjectsHelper "xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}" end + def link_to_bfg + link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer' + end + def legacy_render_context(params) params[:legacy_render] ? { markdown_engine: :redcarpet } : {} end diff --git a/app/mailers/emails/projects.rb b/app/mailers/emails/projects.rb index d7e6c2ba7b2..2500622caa7 100644 --- a/app/mailers/emails/projects.rb +++ b/app/mailers/emails/projects.rb @@ -24,6 +24,21 @@ module Emails subject: subject("Project export error")) end + def repository_cleanup_success_email(project, user) + @project = project + @user = user + + mail(to: user.notification_email, subject: subject("Project cleanup has completed")) + end + + def repository_cleanup_failure_email(project, user, error) + @project = project + @user = user + @error = error + + mail(to: user.notification_email, subject: subject("Project cleanup failure")) + end + def repository_push_email(project_id, opts = {}) @message = Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts) diff --git a/app/models/project.rb b/app/models/project.rb index 1adcb73806d..9e736a3b03c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -339,6 +339,7 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } validates :variables, variable_duplicates: { scope: :environment_scope } + validates :bfg_object_map, file_size: { maximum: :max_attachment_size } # Scopes scope :pending_delete, -> { where(pending_delete: true) } @@ -412,6 +413,9 @@ class Project < ActiveRecord::Base only_integer: true, message: 'needs to be beetween 10 minutes and 1 month' } + # Used by Projects::CleanupService to hold a map of rewritten object IDs + mount_uploader :bfg_object_map, AttachmentUploader + # Returns a project, if it is not about to be removed. # # id - The ID of the project to retrieve. @@ -1973,6 +1977,10 @@ class Project < ActiveRecord::Base Ability.allowed?(user, :read_project_snippet, self) end + def max_attachment_size + Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i + end + private def use_hashed_storage diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index 5904bfbf88d..e24ef7f9c87 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -466,6 +466,14 @@ class NotificationService end end + def repository_cleanup_success(project, user) + mailer.send(:repository_cleanup_success_email, project, user).deliver_later + end + + def repository_cleanup_failure(project, user, error) + mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later + end + protected def new_resource_email(target, method) diff --git a/app/services/projects/cleanup_service.rb b/app/services/projects/cleanup_service.rb new file mode 100644 index 00000000000..12103ea34b5 --- /dev/null +++ b/app/services/projects/cleanup_service.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Projects + # The CleanupService removes data from the project repository following a + # BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/ + # + # Before executing this service, all refs rewritten by BFG should have been + # pushed to the repository + class CleanupService < BaseService + NoUploadError = StandardError.new("Couldn't find uploaded object map") + + include Gitlab::Utils::StrongMemoize + + # Attempt to clean up the project following the push. Warning: this is + # destructive! + # + # path is the path of an upload of a BFG object map file. It contains a line + # per rewritten object, with the old and new SHAs space-separated. It can be + # used to update or remove content that references the objects that BFG has + # altered + # + # Currently, only the project repository is modified by this service, but we + # may wish to modify other data sources in the future. + def execute + apply_bfg_object_map! + + # Remove older objects that are no longer referenced + GitGarbageCollectWorker.new.perform(project.id, :gc) + + # The cache may now be inaccurate, and holding onto it could prevent + # bugs assuming the presence of some object from manifesting for some + # time. Better to feel the pain immediately. + project.repository.expire_all_method_caches + + project.bfg_object_map.remove! + end + + private + + def apply_bfg_object_map! + raise NoUploadError unless project.bfg_object_map.exists? + + project.bfg_object_map.open do |io| + repository_cleaner.apply_bfg_object_map(io) + end + end + + def repository_cleaner + @repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw) + end + end +end diff --git a/app/views/errors/access_denied.html.haml b/app/views/errors/access_denied.html.haml index 8ae29b9d337..46931b5932d 100644 --- a/app/views/errors/access_denied.html.haml +++ b/app/views/errors/access_denied.html.haml @@ -9,7 +9,7 @@ %p = message %p - = s_('403|Please contact your GitLab administrator to get the permission.') + = s_('403|Please contact your GitLab administrator to get permission.') .action-container.js-go-back{ style: 'display: none' } %a{ href: 'javascript:history.back()', class: 'btn btn-success' } = s_('Go Back') diff --git a/app/views/notify/repository_cleanup_failure_email.text.erb b/app/views/notify/repository_cleanup_failure_email.text.erb new file mode 100644 index 00000000000..f5a426a51d1 --- /dev/null +++ b/app/views/notify/repository_cleanup_failure_email.text.erb @@ -0,0 +1,3 @@ +Repository cleanup failed on <%= @project.web_url %> + +<%= @error %> diff --git a/app/views/notify/repository_cleanup_success_email.text.erb b/app/views/notify/repository_cleanup_success_email.text.erb new file mode 100644 index 00000000000..e6e95da2fcc --- /dev/null +++ b/app/views/notify/repository_cleanup_success_email.text.erb @@ -0,0 +1,3 @@ +Repository cleanup succeeded on <%= @project.web_url %> + +Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB diff --git a/app/views/projects/cleanup/_show.html.haml b/app/views/projects/cleanup/_show.html.haml new file mode 100644 index 00000000000..778d27fc61d --- /dev/null +++ b/app/views/projects/cleanup/_show.html.haml @@ -0,0 +1,31 @@ +- return unless Feature.enabled?(:project_cleanup, @project) + +- expanded = Rails.env.test? + +%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) } + .settings-header + %h4= _('Repository cleanup') + %button.btn.js-settings-toggle + = expanded ? _('Collapse') : _('Expand') + %p + = _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe + = link_to icon('question-circle'), + help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'), + target: '_blank', rel: 'noopener noreferrer' + + .settings-content + - url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project) + = form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f| + %fieldset.prepend-top-0.append-bottom-10 + .append-bottom-10 + %h5.prepend-top-0 + = _("Upload object map") + %button.btn.btn-default.js-choose-file{ type: "button" } + = _("Choose a file") + %span.prepend-left-default.js-filename + = _("No file selected") + = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true + .form-text.text-muted + = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size } + = f.submit _('Start cleanup'), class: 'btn btn-success' + diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index f376df29878..1b52821af15 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -53,7 +53,7 @@ = _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git } .prepend-top-5.append-bottom-10 %button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...") - %span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen") + %span.file_name.prepend-left-default.js-filename= _("No file chosen") = f.file_field :avatar, class: "js-project-avatar-input hidden" .form-text.text-muted= _("The maximum file size allowed is 200KB.") - if @project.avatar? diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index c14e95a382c..cb3a035c49e 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -13,3 +13,4 @@ = render "projects/protected_tags/index" = render @deploy_keys = render "projects/deploy_tokens/index" += render "projects/cleanup/show" diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index e51da79c6b5..2c55806a286 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -133,3 +133,4 @@ - create_note_diff_file - delete_diff_files - detect_repository_languages +- repository_cleanup diff --git a/app/workers/repository_cleanup_worker.rb b/app/workers/repository_cleanup_worker.rb new file mode 100644 index 00000000000..aa26c173a72 --- /dev/null +++ b/app/workers/repository_cleanup_worker.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class RepositoryCleanupWorker + include ApplicationWorker + + sidekiq_options retry: 3 + + sidekiq_retries_exhausted do |msg, err| + next if err.is_a?(ActiveRecord::RecordNotFound) + + args = msg['args'] + [msg['error_message']] + + new.perform_failure(*args) + end + + def perform(project_id, user_id) + project = Project.find(project_id) + user = User.find(user_id) + + Projects::CleanupService.new(project, user).execute + + notification_service.repository_cleanup_success(project, user) + end + + def perform_failure(project_id, user_id, error) + project = Project.find(project_id) + user = User.find(user_id) + + # Ensure the file is removed + project.bfg_object_map.remove! + notification_service.repository_cleanup_failure(project, user, error) + end + + private + + def notification_service + @notification_service ||= NotificationService.new + end +end diff --git a/changelogs/unreleased/19376-post-bfg-cleanup.yml b/changelogs/unreleased/19376-post-bfg-cleanup.yml new file mode 100644 index 00000000000..fc1bcc30db9 --- /dev/null +++ b/changelogs/unreleased/19376-post-bfg-cleanup.yml @@ -0,0 +1,5 @@ +--- +title: Use BFG object maps to clean projects +merge_request: 23189 +author: +type: added diff --git a/config/routes/project.rb b/config/routes/project.rb index 3f1ad90dfca..250945cbaa2 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -432,6 +432,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resource :integrations, only: [:show] resource :repository, only: [:show], controller: :repository do post :create_deploy_token, path: 'deploy_token/create' + post :cleanup end end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 53e1c8778b6..d8002815bac 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -81,3 +81,4 @@ - [delete_diff_files, 1] - [detect_repository_languages, 1] - [auto_devops, 2] + - [repository_cleanup, 1] diff --git a/db/migrate/20181203002526_add_project_bfg_object_map_column.rb b/db/migrate/20181203002526_add_project_bfg_object_map_column.rb new file mode 100644 index 00000000000..8b42cd6f941 --- /dev/null +++ b/db/migrate/20181203002526_add_project_bfg_object_map_column.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectBfgObjectMapColumn < ActiveRecord::Migration[5.0] + DOWNTIME = false + + def change + add_column :projects, :bfg_object_map, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 5bc7c7c71fc..d7124100621 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20181129104944) do +ActiveRecord::Schema.define(version: 20181203002526) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -1684,6 +1684,7 @@ ActiveRecord::Schema.define(version: 20181129104944) do t.boolean "remote_mirror_available_overridden" t.bigint "pool_repository_id" t.string "runners_token_encrypted" + t.string "bfg_object_map" t.index ["ci_id"], name: "index_projects_on_ci_id", using: :btree t.index ["created_at"], name: "index_projects_on_created_at", using: :btree t.index ["creator_id"], name: "index_projects_on_creator_id", using: :btree diff --git a/doc/user/project/repository/img/repository_cleanup.png b/doc/user/project/repository/img/repository_cleanup.png Binary files differnew file mode 100644 index 00000000000..2749392ffa4 --- /dev/null +++ b/doc/user/project/repository/img/repository_cleanup.png diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index d534c8cbe4b..672567a8d7d 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -1,43 +1,105 @@ # Reducing the repository size using Git A GitLab Enterprise Edition administrator can set a [repository size limit][admin-repo-size] -which will prevent you to exceed it. +which will prevent you from exceeding it. When a project has reached its size limit, you will not be able to push to it, create a new merge request, or merge existing ones. You will still be able to create new issues, and clone the project though. Uploading LFS objects will also be denied. -In order to lift these restrictions, the administrator of the GitLab instance -needs to increase the limit on the particular project that exceeded it or you -need to instruct Git to rewrite changes. - If you exceed the repository size limit, your first thought might be to remove -some data, make a new commit and push back to the repository. Unfortunately, -it's not so easy and that workflow won't work. Deleting files in a commit doesn't -actually reduce the size of the repo since the earlier commits and blobs are -still around. What you need to do is rewrite history with Git's -[`filter-branch` option][gitscm]. +some data, make a new commit and push back to the repository. Perhaps you can +move some blobs to LFS, or remove some old dependency updates from history. +Unfortunately, it's not so easy and that workflow won't work. Deleting files in +a commit doesn't actually reduce the size of the repo since the earlier commits +and blobs are still around. What you need to do is rewrite history with Git's +[`filter-branch` option][gitscm], or a tool like the [BFG Repo-Cleaner][bfg]. Note that even with that method, until `git gc` runs on the GitLab side, the -"removed" commits and blobs will still be around. And if a commit was ever -included in an MR, or if a build was run for a commit, or if a user commented -on it, it will be kept around too. So, in these cases the size will not decrease. - -The only fool proof way to actually decrease the repository size is to prune all -the unneeded stuff locally, and then create a new project on GitLab and start -using that instead. +"removed" commits and blobs will still be around. You also need to be able to +push the rewritten history to GitLab, which may be impossible if you've already +exceeded the maximum size limit. -With that being said, you can try reducing your repository size with the -following method. - -## Using `git filter-branch` to purge files +In order to lift these restrictions, the administrator of the GitLab instance +needs to increase the limit on the particular project that exceeded it, so it's +always better to spot that you're approaching the limit and act proactively to +stay underneath it. If you hit the limit, and your admin can't - or won't - +temporarily increase it for you, your only option is to prune all the unneeded +stuff locally, and then create a new project on GitLab and start using that +instead. + +If you can continue to use the original project, we recommend [using the +BFG Repo-Cleaner](#using-the-bfg-repo-cleaner). It's faster and simpler than +`git filter-branch`, and GitLab can use its account of what has changed to clean +up its own internal state, maximizing the space saved. > **Warning:** > Make sure to first make a copy of your repository since rewriting history will > purge the files and information you are about to delete. Also make sure to > inform any collaborators to not use `pull` after your changes, but use `rebase`. +> **Warning:** +> This process is not suitable for removing sensitive data like password or keys +> from your repository. Information about commits, including file content, is +> cached in the database, and will remain visible even after they have been +> removed from the repository. + +## Using the BFG Repo-Cleaner + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/19376) in GitLab 11.6. + +1. [Install BFG](https://rtyley.github.io/bfg-repo-cleaner/). + +1. Navigate to your repository: + + ``` + cd my_repository/ + ``` + +1. Change to the branch you want to remove the big file from: + + ``` + git checkout master + ``` + +1. Create a commit removing the large file from the branch, if it still exists: + + ``` + git rm path/to/big_file.mpg + git commit -m 'Remove unneeded large file' + ``` + +1. Rewrite history: + + ``` + bfg --delete-files path/to/big_file.mpg + ``` + + An object map file will be written to `object-id-map.old-new.txt`. Keep it + around - you'll need it for the final step! + +1. Force-push the changes to GitLab: + + ``` + git push --force-with-lease origin master + ``` + + If this step fails, someone has changed the `master` branch while you were + rewriting history. You could restore the branch and re-run BFG to preserve + their changes, or use `git push --force` to overwrite their changes. + +1. Navigate to **Project > Settings > Repository > Repository Cleanup**: + +  + + Upload the `object-id-map.old-new.txt` file and press **Start cleanup**. + This will remove any internal git references to the old commits, and run + `git gc` against the repository. You will receive an email once it has + completed. + +## Using `git filter-branch` + 1. Navigate to your repository: ``` @@ -70,11 +132,6 @@ following method. Your repository should now be below the size limit. -> **Note:** -> As an alternative to `filter-branch`, you can use the `bfg` tool with a -> command like: `bfg --delete-files path/to/big_file.mpg`. Read the -> [BFG Repo-Cleaner][bfg] documentation for more information. - [admin-repo-size]: https://docs.gitlab.com/ee/user/admin_area/settings/account_and_limit_settings.html#repository-size-limit [bfg]: https://rtyley.github.io/bfg-repo-cleaner/ [gitscm]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#The-Nuclear-Option:-filter-branch diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb new file mode 100644 index 00000000000..2d1d8435cf3 --- /dev/null +++ b/lib/gitlab/git/repository_cleaner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class RepositoryCleaner + include Gitlab::Git::WrapsGitalyErrors + + attr_reader :repository + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + end + + def apply_bfg_object_map(io) + wrapped_gitaly_errors do + gitaly_cleanup_client.apply_bfg_object_map(io) + end + end + + private + + def gitaly_cleanup_client + @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb new file mode 100644 index 00000000000..8e412a9b3ef --- /dev/null +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class CleanupService + attr_reader :repository, :gitaly_repo, :storage + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def apply_bfg_object_map(io) + first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo) + + enum = Enumerator.new do |y| + y.yield first_request + + while data = io.read(RepositoryService::MAX_MSG_SIZE) + y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + end + end + + GitalyClient.call( + storage, + :cleanup_service, + :apply_bfg_object_map, + enum, + timeout: GitalyClient.no_timeout + ) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 7cdea9d1ce4..d10d4f2f746 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -115,6 +115,7 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + - :bfg_object_map namespaces: - :runners_token - :runners_token_encrypted diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 23ee90ff0dd..2b9beea17ce 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -225,7 +225,7 @@ msgstr "" msgid "2FA enabled" msgstr "" -msgid "403|Please contact your GitLab administrator to get the permission." +msgid "403|Please contact your GitLab administrator to get permission." msgstr "" msgid "403|You don't have the permission to access this page." @@ -1244,6 +1244,9 @@ msgstr "" msgid "Choose a branch/tag (e.g. %{master}) or enter a commit (e.g. %{sha}) to see what's changed or to create a merge request." msgstr "" +msgid "Choose a file" +msgstr "" + msgid "Choose a template..." msgstr "" @@ -2939,6 +2942,9 @@ msgstr "" msgid "Failed to update issues, please try again." msgstr "" +msgid "Failed to upload object map file" +msgstr "" + msgid "Failure" msgstr "" @@ -4333,6 +4339,9 @@ msgstr "" msgid "No file chosen" msgstr "" +msgid "No file selected" +msgstr "" + msgid "No files found." msgstr "" @@ -5489,6 +5498,12 @@ msgstr "" msgid "Repository URL" msgstr "" +msgid "Repository cleanup" +msgstr "" + +msgid "Repository cleanup has started. You will receive an email once the cleanup operation is complete." +msgstr "" + msgid "Repository maintenance" msgstr "" @@ -6162,6 +6177,9 @@ msgstr "" msgid "Start and due date" msgstr "" +msgid "Start cleanup" +msgstr "" + msgid "Start date" msgstr "" @@ -6377,6 +6395,9 @@ msgstr "" msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage." msgstr "" +msgid "The maximum file size allowed is %{max_attachment_size}mb" +msgstr "" + msgid "The maximum file size allowed is 200KB." msgstr "" @@ -7059,6 +7080,9 @@ msgstr "" msgid "Upload file" msgstr "" +msgid "Upload object map" +msgstr "" + msgid "UploadLink|click to upload" msgstr "" diff --git a/spec/controllers/projects/settings/repository_controller_spec.rb b/spec/controllers/projects/settings/repository_controller_spec.rb index 9cee40b7553..69ec971bb75 100644 --- a/spec/controllers/projects/settings/repository_controller_spec.rb +++ b/spec/controllers/projects/settings/repository_controller_spec.rb @@ -17,4 +17,35 @@ describe Projects::Settings::RepositoryController do expect(response).to render_template(:show) end end + + describe 'PUT cleanup' do + def do_put! + object_map = fixture_file_upload('spec/fixtures/bfg_object_map.txt') + + Sidekiq::Testing.fake! do + put :cleanup, namespace_id: project.namespace, project_id: project, project: { object_map: object_map } + end + end + + context 'feature enabled' do + it 'enqueues a RepositoryCleanupWorker' do + stub_feature_flags(project_cleanup: true) + + do_put! + + expect(response).to redirect_to project_settings_repository_path(project) + expect(RepositoryCleanupWorker.jobs.count).to eq(1) + end + end + + context 'feature disabled' do + it 'shows a 404 error' do + stub_feature_flags(project_cleanup: false) + + do_put! + + expect(response).to have_gitlab_http_status(404) + end + end + end end diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb index b7a22316d26..418e22f8c35 100644 --- a/spec/features/projects/settings/repository_settings_spec.rb +++ b/spec/features/projects/settings/repository_settings_spec.rb @@ -196,5 +196,40 @@ describe 'Projects > Settings > Repository settings' do end end end + + context 'repository cleanup settings' do + let(:object_map_file) { Rails.root.join('spec', 'fixtures', 'bfg_object_map.txt') } + + context 'feature enabled' do + it 'uploads an object map file', :js do + stub_feature_flags(project_cleanup: true) + + visit project_settings_repository_path(project) + + expect(page).to have_content('Repository cleanup') + + page.within('#cleanup') do + attach_file('project[bfg_object_map]', object_map_file, visible: false) + + Sidekiq::Testing.fake! do + click_button 'Start cleanup' + end + end + + expect(page).to have_content('Repository cleanup has started') + expect(RepositoryCleanupWorker.jobs.count).to eq(1) + end + end + + context 'feature disabled' do + it 'does not show the settings' do + stub_feature_flags(project_cleanup: false) + + visit project_settings_repository_path(project) + + expect(page).not_to have_content('Repository cleanup') + end + end + end end end diff --git a/spec/fixtures/bfg_object_map.txt b/spec/fixtures/bfg_object_map.txt new file mode 100644 index 00000000000..c60171d8770 --- /dev/null +++ b/spec/fixtures/bfg_object_map.txt @@ -0,0 +1 @@ +f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 e242ed3bffccdf271b7fbaf34ed72d089537b42f diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 976b6c312b4..a857b7646b2 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -471,6 +471,31 @@ describe ProjectsHelper do end end + describe 'link_to_bfg' do + subject { helper.link_to_bfg } + + it 'generates a hardcoded link to the BFG Repo-Cleaner' do + result = helper.link_to_bfg + doc = Nokogiri::HTML.fragment(result) + + expect(doc.children.size).to eq(1) + + link = doc.children.first + + aggregate_failures do + expect(result).to be_html_safe + + expect(link.name).to eq('a') + expect(link[:target]).to eq('_blank') + expect(link[:rel]).to eq('noopener noreferrer') + expect(link[:href]).to eq('https://rtyley.github.io/bfg-repo-cleaner/') + expect(link.inner_html).to eq('BFG') + + expect(result).to be_html_safe + end + end + end + describe '#legacy_render_context' do it 'returns the redcarpet engine' do params = { legacy_render: '1' } diff --git a/spec/javascripts/lib/utils/file_upload_spec.js b/spec/javascripts/lib/utils/file_upload_spec.js new file mode 100644 index 00000000000..92c9cc70aaf --- /dev/null +++ b/spec/javascripts/lib/utils/file_upload_spec.js @@ -0,0 +1,36 @@ +import fileUpload from '~/lib/utils/file_upload'; + +describe('File upload', () => { + beforeEach(() => { + setFixtures(` + <form> + <button class="js-button" type="button">Click me!</button> + <input type="text" class="js-input" /> + <span class="js-filename"></span> + </form> + `); + + fileUpload('.js-button', '.js-input'); + }); + + it('clicks file input after clicking button', () => { + const btn = document.querySelector('.js-button'); + const input = document.querySelector('.js-input'); + + spyOn(input, 'click'); + + btn.click(); + + expect(input.click).toHaveBeenCalled(); + }); + + it('updates file name text', () => { + const input = document.querySelector('.js-input'); + + input.value = 'path/to/file/index.js'; + + input.dispatchEvent(new CustomEvent('change')); + + expect(document.querySelector('.js-filename').textContent).toEqual('index.js'); + }); +}); diff --git a/spec/lib/gitlab/git/repository_cleaner_spec.rb b/spec/lib/gitlab/git/repository_cleaner_spec.rb new file mode 100644 index 00000000000..a9d9e67ef94 --- /dev/null +++ b/spec/lib/gitlab/git/repository_cleaner_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe Gitlab::Git::RepositoryCleaner do + let(:project) { create(:project, :repository) } + let(:repository) { project.repository } + let(:head_sha) { repository.head_commit.id } + + let(:object_map) { StringIO.new("#{head_sha} #{'0' * 40}") } + + subject(:cleaner) { described_class.new(repository.raw) } + + describe '#apply_bfg_object_map' do + it 'removes internal references pointing at SHAs in the object map' do + # Create some refs we expect to be removed + repository.keep_around(head_sha) + repository.create_ref(head_sha, 'refs/environments/1') + repository.create_ref(head_sha, 'refs/merge-requests/1') + repository.create_ref(head_sha, 'refs/heads/_keep') + repository.create_ref(head_sha, 'refs/tags/_keep') + + cleaner.apply_bfg_object_map(object_map) + + aggregate_failures do + expect(repository.kept_around?(head_sha)).to be_falsy + expect(repository.ref_exists?('refs/environments/1')).to be_falsy + expect(repository.ref_exists?('refs/merge-requests/1')).to be_falsy + expect(repository.ref_exists?('refs/heads/_keep')).to be_truthy + expect(repository.ref_exists?('refs/tags/_keep')).to be_truthy + end + end + end +end diff --git a/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb new file mode 100644 index 00000000000..369deff732a --- /dev/null +++ b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Gitlab::GitalyClient::CleanupService do + let(:project) { create(:project) } + let(:storage_name) { project.repository_storage } + let(:relative_path) { project.disk_path + '.git' } + let(:client) { described_class.new(project.repository) } + + describe '#apply_bfg_object_map' do + it 'sends an apply_bfg_object_map message' do + expect_any_instance_of(Gitaly::CleanupService::Stub) + .to receive(:apply_bfg_object_map) + .with(kind_of(Enumerator), kind_of(Hash)) + .and_return(double) + + client.apply_bfg_object_map(StringIO.new) + end + end +end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index 2d8da7673dc..0f6c2604984 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -2146,6 +2146,27 @@ describe NotificationService, :mailer do end end + describe 'Repository cleanup' do + let(:user) { create(:user) } + let(:project) { create(:project) } + + describe '#repository_cleanup_success' do + it 'emails the specified user only' do + notification.repository_cleanup_success(project, user) + + should_email(user) + end + end + + describe '#repository_cleanup_failure' do + it 'emails the specified user only' do + notification.repository_cleanup_failure(project, user, 'Some error') + + should_email(user) + end + end + end + def build_team(project) @u_watcher = create_global_setting_for(create(:user), :watch) @u_participating = create_global_setting_for(create(:user), :participating) diff --git a/spec/services/projects/cleanup_service_spec.rb b/spec/services/projects/cleanup_service_spec.rb new file mode 100644 index 00000000000..3d4587ce2a1 --- /dev/null +++ b/spec/services/projects/cleanup_service_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' + +describe Projects::CleanupService do + let(:project) { create(:project, :repository, bfg_object_map: fixture_file_upload('spec/fixtures/bfg_object_map.txt')) } + let(:object_map) { project.bfg_object_map } + + subject(:service) { described_class.new(project) } + + describe '#execute' do + it 'runs the apply_bfg_object_map gitaly RPC' do + expect_next_instance_of(Gitlab::Git::RepositoryCleaner) do |cleaner| + expect(cleaner).to receive(:apply_bfg_object_map).with(kind_of(IO)) + end + + service.execute + end + + it 'runs garbage collection on the repository' do + expect_next_instance_of(GitGarbageCollectWorker) do |worker| + expect(worker).to receive(:perform) + end + + service.execute + end + + it 'clears the repository cache' do + expect(project.repository).to receive(:expire_all_method_caches) + + service.execute + end + + it 'removes the object map file' do + service.execute + + expect(object_map.exists?).to be_falsy + end + + it 'raises an error if no object map can be found' do + object_map.remove! + + expect { service.execute }.to raise_error(described_class::NoUploadError) + end + end +end diff --git a/spec/workers/repository_cleanup_worker_spec.rb b/spec/workers/repository_cleanup_worker_spec.rb new file mode 100644 index 00000000000..3adae0b6cfa --- /dev/null +++ b/spec/workers/repository_cleanup_worker_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +describe RepositoryCleanupWorker do + let(:project) { create(:project) } + let(:user) { create(:user) } + + subject(:worker) { described_class.new } + + describe '#perform' do + it 'executes the cleanup service and sends a success notification' do + expect_next_instance_of(Projects::CleanupService) do |service| + expect(service.project).to eq(project) + expect(service.current_user).to eq(user) + + expect(service).to receive(:execute) + end + + expect_next_instance_of(NotificationService) do |service| + expect(service).to receive(:repository_cleanup_success).with(project, user) + end + + worker.perform(project.id, user.id) + end + + it 'raises an error if the project cannot be found' do + project.destroy + + expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'raises an error if the user cannot be found' do + user.destroy + + expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + describe '#sidekiq_retries_exhausted' do + let(:job) { { 'args' => [project.id, user.id], 'error_message' => 'Error' } } + + it 'does not send a failure notification for a RecordNotFound error' do + expect(NotificationService).not_to receive(:new) + + described_class.sidekiq_retries_exhausted_block.call(job, ActiveRecord::RecordNotFound.new) + end + + it 'sends a failure notification' do + expect_next_instance_of(NotificationService) do |service| + expect(service).to receive(:repository_cleanup_failure).with(project, user, 'Error') + end + + described_class.sidekiq_retries_exhausted_block.call(job, StandardError.new) + end + end +end |
