summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/assets/javascripts/behaviors/requires_input.js7
-rw-r--r--app/assets/javascripts/lib/utils/file_upload.js13
-rw-r--r--app/assets/javascripts/pages/projects/edit/index.js4
-rw-r--r--app/assets/javascripts/pages/projects/settings/repository/form.js2
-rw-r--r--app/assets/javascripts/pages/projects/shared/project_avatar.js16
-rw-r--r--app/controllers/projects/settings/repository_controller.rb19
-rw-r--r--app/helpers/projects_helper.rb4
-rw-r--r--app/mailers/emails/projects.rb15
-rw-r--r--app/models/project.rb8
-rw-r--r--app/services/notification_service.rb8
-rw-r--r--app/services/projects/cleanup_service.rb52
-rw-r--r--app/views/errors/access_denied.html.haml2
-rw-r--r--app/views/notify/repository_cleanup_failure_email.text.erb3
-rw-r--r--app/views/notify/repository_cleanup_success_email.text.erb3
-rw-r--r--app/views/projects/cleanup/_show.html.haml31
-rw-r--r--app/views/projects/edit.html.haml2
-rw-r--r--app/views/projects/settings/repository/show.html.haml1
-rw-r--r--app/workers/all_queues.yml1
-rw-r--r--app/workers/repository_cleanup_worker.rb39
-rw-r--r--changelogs/unreleased/19376-post-bfg-cleanup.yml5
-rw-r--r--config/routes/project.rb1
-rw-r--r--config/sidekiq_queues.yml1
-rw-r--r--db/migrate/20181203002526_add_project_bfg_object_map_column.rb9
-rw-r--r--db/schema.rb3
-rw-r--r--doc/user/project/repository/img/repository_cleanup.pngbin0 -> 20833 bytes
-rw-r--r--doc/user/project/repository/reducing_the_repo_size_using_git.md109
-rw-r--r--lib/gitlab/git/repository_cleaner.rb28
-rw-r--r--lib/gitlab/gitaly_client/cleanup_service.rb36
-rw-r--r--lib/gitlab/import_export/import_export.yml1
-rw-r--r--locale/gitlab.pot26
-rw-r--r--spec/controllers/projects/settings/repository_controller_spec.rb31
-rw-r--r--spec/features/projects/settings/repository_settings_spec.rb35
-rw-r--r--spec/fixtures/bfg_object_map.txt1
-rw-r--r--spec/helpers/projects_helper_spec.rb25
-rw-r--r--spec/javascripts/lib/utils/file_upload_spec.js36
-rw-r--r--spec/lib/gitlab/git/repository_cleaner_spec.rb32
-rw-r--r--spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb19
-rw-r--r--spec/services/notification_service_spec.rb21
-rw-r--r--spec/services/projects/cleanup_service_spec.rb44
-rw-r--r--spec/workers/repository_cleanup_worker_spec.rb55
40 files changed, 697 insertions, 51 deletions
diff --git a/app/assets/javascripts/behaviors/requires_input.js b/app/assets/javascripts/behaviors/requires_input.js
index c09d9ccddd6..d8056e48d4e 100644
--- a/app/assets/javascripts/behaviors/requires_input.js
+++ b/app/assets/javascripts/behaviors/requires_input.js
@@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) {
}
$(() => {
- const $form = $('form.js-requires-input');
- if ($form) {
+ $('form.js-requires-input').each((i, el) => {
+ const $form = $(el);
+
$form.requiresInput();
hideOrShowHelpBlock($form);
$('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form));
- }
+ });
});
diff --git a/app/assets/javascripts/lib/utils/file_upload.js b/app/assets/javascripts/lib/utils/file_upload.js
new file mode 100644
index 00000000000..b41ffb44971
--- /dev/null
+++ b/app/assets/javascripts/lib/utils/file_upload.js
@@ -0,0 +1,13 @@
+export default (buttonSelector, fileSelector) => {
+ const btn = document.querySelector(buttonSelector);
+ const fileInput = document.querySelector(fileSelector);
+ const form = btn.closest('form');
+
+ btn.addEventListener('click', () => {
+ fileInput.click();
+ });
+
+ fileInput.addEventListener('change', () => {
+ form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
+ });
+};
diff --git a/app/assets/javascripts/pages/projects/edit/index.js b/app/assets/javascripts/pages/projects/edit/index.js
index f5b1cf85e68..899d5925956 100644
--- a/app/assets/javascripts/pages/projects/edit/index.js
+++ b/app/assets/javascripts/pages/projects/edit/index.js
@@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels';
import setupProjectEdit from '~/project_edit';
import initConfirmDangerModal from '~/confirm_danger_modal';
import mountBadgeSettings from '~/pages/shared/mount_badge_settings';
+import fileUpload from '~/lib/utils/file_upload';
import initProjectLoadingSpinner from '../shared/save_project_loader';
-import projectAvatar from '../shared/project_avatar';
import initProjectPermissionsSettings from '../shared/permissions';
document.addEventListener('DOMContentLoaded', () => {
@@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => {
setupProjectEdit();
// Initialize expandable settings panels
initSettingsPanels();
- projectAvatar();
+ fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input');
initProjectPermissionsSettings();
initConfirmDangerModal();
mountBadgeSettings(PROJECT_BADGE);
diff --git a/app/assets/javascripts/pages/projects/settings/repository/form.js b/app/assets/javascripts/pages/projects/settings/repository/form.js
index a52861c9efa..3e02893f24c 100644
--- a/app/assets/javascripts/pages/projects/settings/repository/form.js
+++ b/app/assets/javascripts/pages/projects/settings/repository/form.js
@@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys';
import ProtectedBranchCreate from '~/protected_branches/protected_branch_create';
import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list';
import DueDateSelectors from '~/due_date_select';
+import fileUpload from '~/lib/utils/file_upload';
export default () => {
new ProtectedTagCreate();
@@ -16,4 +17,5 @@ export default () => {
new ProtectedBranchCreate();
new ProtectedBranchEditList();
new DueDateSelectors();
+ fileUpload('.js-choose-file', '.js-object-map-input');
};
diff --git a/app/assets/javascripts/pages/projects/shared/project_avatar.js b/app/assets/javascripts/pages/projects/shared/project_avatar.js
deleted file mode 100644
index 1e69ecb481d..00000000000
--- a/app/assets/javascripts/pages/projects/shared/project_avatar.js
+++ /dev/null
@@ -1,16 +0,0 @@
-import $ from 'jquery';
-
-export default function projectAvatar() {
- $('.js-choose-project-avatar-button').bind('click', function onClickAvatar() {
- const form = $(this).closest('form');
- return form.find('.js-project-avatar-input').click();
- });
-
- $('.js-project-avatar-input').bind('change', function onClickAvatarInput() {
- const form = $(this).closest('form');
- const filename = $(this)
- .val()
- .replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
- return form.find('.js-avatar-filename').text(filename);
- });
-}
diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb
index 1d76c90d4eb..30724de7f6a 100644
--- a/app/controllers/projects/settings/repository_controller.rb
+++ b/app/controllers/projects/settings/repository_controller.rb
@@ -5,6 +5,7 @@ module Projects
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
before_action :remote_mirror, only: [:show]
+ before_action :check_cleanup_feature_flag!, only: :cleanup
def show
render_show
@@ -20,8 +21,26 @@ module Projects
render_show
end
+ def cleanup
+ cleanup_params = params.require(:project).permit(:bfg_object_map)
+ result = Projects::UpdateService.new(project, current_user, cleanup_params).execute
+
+ if result[:status] == :success
+ RepositoryCleanupWorker.perform_async(project.id, current_user.id)
+ flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.')
+ else
+ flash[:alert] = _('Failed to upload object map file')
+ end
+
+ redirect_to project_settings_repository_path(project)
+ end
+
private
+ def check_cleanup_feature_flag!
+ render_404 unless ::Feature.enabled?(:project_cleanup, project)
+ end
+
def render_show
@deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user)
@deploy_tokens = @project.deploy_tokens.active
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb
index 0a7f930110a..fcec6d2c9d3 100644
--- a/app/helpers/projects_helper.rb
+++ b/app/helpers/projects_helper.rb
@@ -257,6 +257,10 @@ module ProjectsHelper
"xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
end
+ def link_to_bfg
+ link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer'
+ end
+
def legacy_render_context(params)
params[:legacy_render] ? { markdown_engine: :redcarpet } : {}
end
diff --git a/app/mailers/emails/projects.rb b/app/mailers/emails/projects.rb
index d7e6c2ba7b2..2500622caa7 100644
--- a/app/mailers/emails/projects.rb
+++ b/app/mailers/emails/projects.rb
@@ -24,6 +24,21 @@ module Emails
subject: subject("Project export error"))
end
+ def repository_cleanup_success_email(project, user)
+ @project = project
+ @user = user
+
+ mail(to: user.notification_email, subject: subject("Project cleanup has completed"))
+ end
+
+ def repository_cleanup_failure_email(project, user, error)
+ @project = project
+ @user = user
+ @error = error
+
+ mail(to: user.notification_email, subject: subject("Project cleanup failure"))
+ end
+
def repository_push_email(project_id, opts = {})
@message =
Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)
diff --git a/app/models/project.rb b/app/models/project.rb
index 1adcb73806d..9e736a3b03c 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -339,6 +339,7 @@ class Project < ActiveRecord::Base
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
validates :variables, variable_duplicates: { scope: :environment_scope }
+ validates :bfg_object_map, file_size: { maximum: :max_attachment_size }
# Scopes
scope :pending_delete, -> { where(pending_delete: true) }
@@ -412,6 +413,9 @@ class Project < ActiveRecord::Base
only_integer: true,
message: 'needs to be beetween 10 minutes and 1 month' }
+ # Used by Projects::CleanupService to hold a map of rewritten object IDs
+ mount_uploader :bfg_object_map, AttachmentUploader
+
# Returns a project, if it is not about to be removed.
#
# id - The ID of the project to retrieve.
@@ -1973,6 +1977,10 @@ class Project < ActiveRecord::Base
Ability.allowed?(user, :read_project_snippet, self)
end
+ def max_attachment_size
+ Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i
+ end
+
private
def use_hashed_storage
diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb
index 5904bfbf88d..e24ef7f9c87 100644
--- a/app/services/notification_service.rb
+++ b/app/services/notification_service.rb
@@ -466,6 +466,14 @@ class NotificationService
end
end
+ def repository_cleanup_success(project, user)
+ mailer.send(:repository_cleanup_success_email, project, user).deliver_later
+ end
+
+ def repository_cleanup_failure(project, user, error)
+ mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later
+ end
+
protected
def new_resource_email(target, method)
diff --git a/app/services/projects/cleanup_service.rb b/app/services/projects/cleanup_service.rb
new file mode 100644
index 00000000000..12103ea34b5
--- /dev/null
+++ b/app/services/projects/cleanup_service.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Projects
+ # The CleanupService removes data from the project repository following a
+ # BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
+ #
+ # Before executing this service, all refs rewritten by BFG should have been
+ # pushed to the repository
+ class CleanupService < BaseService
+ NoUploadError = StandardError.new("Couldn't find uploaded object map")
+
+ include Gitlab::Utils::StrongMemoize
+
+ # Attempt to clean up the project following the push. Warning: this is
+ # destructive!
+ #
+ # path is the path of an upload of a BFG object map file. It contains a line
+ # per rewritten object, with the old and new SHAs space-separated. It can be
+ # used to update or remove content that references the objects that BFG has
+ # altered
+ #
+ # Currently, only the project repository is modified by this service, but we
+ # may wish to modify other data sources in the future.
+ def execute
+ apply_bfg_object_map!
+
+ # Remove older objects that are no longer referenced
+ GitGarbageCollectWorker.new.perform(project.id, :gc)
+
+ # The cache may now be inaccurate, and holding onto it could prevent
+ # bugs assuming the presence of some object from manifesting for some
+ # time. Better to feel the pain immediately.
+ project.repository.expire_all_method_caches
+
+ project.bfg_object_map.remove!
+ end
+
+ private
+
+ def apply_bfg_object_map!
+ raise NoUploadError unless project.bfg_object_map.exists?
+
+ project.bfg_object_map.open do |io|
+ repository_cleaner.apply_bfg_object_map(io)
+ end
+ end
+
+ def repository_cleaner
+ @repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw)
+ end
+ end
+end
diff --git a/app/views/errors/access_denied.html.haml b/app/views/errors/access_denied.html.haml
index 8ae29b9d337..46931b5932d 100644
--- a/app/views/errors/access_denied.html.haml
+++ b/app/views/errors/access_denied.html.haml
@@ -9,7 +9,7 @@
%p
= message
%p
- = s_('403|Please contact your GitLab administrator to get the permission.')
+ = s_('403|Please contact your GitLab administrator to get permission.')
.action-container.js-go-back{ style: 'display: none' }
%a{ href: 'javascript:history.back()', class: 'btn btn-success' }
= s_('Go Back')
diff --git a/app/views/notify/repository_cleanup_failure_email.text.erb b/app/views/notify/repository_cleanup_failure_email.text.erb
new file mode 100644
index 00000000000..f5a426a51d1
--- /dev/null
+++ b/app/views/notify/repository_cleanup_failure_email.text.erb
@@ -0,0 +1,3 @@
+Repository cleanup failed on <%= @project.web_url %>
+
+<%= @error %>
diff --git a/app/views/notify/repository_cleanup_success_email.text.erb b/app/views/notify/repository_cleanup_success_email.text.erb
new file mode 100644
index 00000000000..e6e95da2fcc
--- /dev/null
+++ b/app/views/notify/repository_cleanup_success_email.text.erb
@@ -0,0 +1,3 @@
+Repository cleanup succeeded on <%= @project.web_url %>
+
+Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB
diff --git a/app/views/projects/cleanup/_show.html.haml b/app/views/projects/cleanup/_show.html.haml
new file mode 100644
index 00000000000..778d27fc61d
--- /dev/null
+++ b/app/views/projects/cleanup/_show.html.haml
@@ -0,0 +1,31 @@
+- return unless Feature.enabled?(:project_cleanup, @project)
+
+- expanded = Rails.env.test?
+
+%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) }
+ .settings-header
+ %h4= _('Repository cleanup')
+ %button.btn.js-settings-toggle
+ = expanded ? _('Collapse') : _('Expand')
+ %p
+ = _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe
+ = link_to icon('question-circle'),
+ help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'),
+ target: '_blank', rel: 'noopener noreferrer'
+
+ .settings-content
+ - url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project)
+ = form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f|
+ %fieldset.prepend-top-0.append-bottom-10
+ .append-bottom-10
+ %h5.prepend-top-0
+ = _("Upload object map")
+ %button.btn.btn-default.js-choose-file{ type: "button" }
+ = _("Choose a file")
+ %span.prepend-left-default.js-filename
+ = _("No file selected")
+ = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
+ .form-text.text-muted
+ = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
+ = f.submit _('Start cleanup'), class: 'btn btn-success'
+
diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml
index f376df29878..1b52821af15 100644
--- a/app/views/projects/edit.html.haml
+++ b/app/views/projects/edit.html.haml
@@ -53,7 +53,7 @@
= _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git }
.prepend-top-5.append-bottom-10
%button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...")
- %span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen")
+ %span.file_name.prepend-left-default.js-filename= _("No file chosen")
= f.file_field :avatar, class: "js-project-avatar-input hidden"
.form-text.text-muted= _("The maximum file size allowed is 200KB.")
- if @project.avatar?
diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml
index c14e95a382c..cb3a035c49e 100644
--- a/app/views/projects/settings/repository/show.html.haml
+++ b/app/views/projects/settings/repository/show.html.haml
@@ -13,3 +13,4 @@
= render "projects/protected_tags/index"
= render @deploy_keys
= render "projects/deploy_tokens/index"
+= render "projects/cleanup/show"
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index e51da79c6b5..2c55806a286 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -133,3 +133,4 @@
- create_note_diff_file
- delete_diff_files
- detect_repository_languages
+- repository_cleanup
diff --git a/app/workers/repository_cleanup_worker.rb b/app/workers/repository_cleanup_worker.rb
new file mode 100644
index 00000000000..aa26c173a72
--- /dev/null
+++ b/app/workers/repository_cleanup_worker.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class RepositoryCleanupWorker
+ include ApplicationWorker
+
+ sidekiq_options retry: 3
+
+ sidekiq_retries_exhausted do |msg, err|
+ next if err.is_a?(ActiveRecord::RecordNotFound)
+
+ args = msg['args'] + [msg['error_message']]
+
+ new.perform_failure(*args)
+ end
+
+ def perform(project_id, user_id)
+ project = Project.find(project_id)
+ user = User.find(user_id)
+
+ Projects::CleanupService.new(project, user).execute
+
+ notification_service.repository_cleanup_success(project, user)
+ end
+
+ def perform_failure(project_id, user_id, error)
+ project = Project.find(project_id)
+ user = User.find(user_id)
+
+ # Ensure the file is removed
+ project.bfg_object_map.remove!
+ notification_service.repository_cleanup_failure(project, user, error)
+ end
+
+ private
+
+ def notification_service
+ @notification_service ||= NotificationService.new
+ end
+end
diff --git a/changelogs/unreleased/19376-post-bfg-cleanup.yml b/changelogs/unreleased/19376-post-bfg-cleanup.yml
new file mode 100644
index 00000000000..fc1bcc30db9
--- /dev/null
+++ b/changelogs/unreleased/19376-post-bfg-cleanup.yml
@@ -0,0 +1,5 @@
+---
+title: Use BFG object maps to clean projects
+merge_request: 23189
+author:
+type: added
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 3f1ad90dfca..250945cbaa2 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -432,6 +432,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
resource :integrations, only: [:show]
resource :repository, only: [:show], controller: :repository do
post :create_deploy_token, path: 'deploy_token/create'
+ post :cleanup
end
end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 53e1c8778b6..d8002815bac 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -81,3 +81,4 @@
- [delete_diff_files, 1]
- [detect_repository_languages, 1]
- [auto_devops, 2]
+ - [repository_cleanup, 1]
diff --git a/db/migrate/20181203002526_add_project_bfg_object_map_column.rb b/db/migrate/20181203002526_add_project_bfg_object_map_column.rb
new file mode 100644
index 00000000000..8b42cd6f941
--- /dev/null
+++ b/db/migrate/20181203002526_add_project_bfg_object_map_column.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddProjectBfgObjectMapColumn < ActiveRecord::Migration[5.0]
+ DOWNTIME = false
+
+ def change
+ add_column :projects, :bfg_object_map, :string
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5bc7c7c71fc..d7124100621 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20181129104944) do
+ActiveRecord::Schema.define(version: 20181203002526) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -1684,6 +1684,7 @@ ActiveRecord::Schema.define(version: 20181129104944) do
t.boolean "remote_mirror_available_overridden"
t.bigint "pool_repository_id"
t.string "runners_token_encrypted"
+ t.string "bfg_object_map"
t.index ["ci_id"], name: "index_projects_on_ci_id", using: :btree
t.index ["created_at"], name: "index_projects_on_created_at", using: :btree
t.index ["creator_id"], name: "index_projects_on_creator_id", using: :btree
diff --git a/doc/user/project/repository/img/repository_cleanup.png b/doc/user/project/repository/img/repository_cleanup.png
new file mode 100644
index 00000000000..2749392ffa4
--- /dev/null
+++ b/doc/user/project/repository/img/repository_cleanup.png
Binary files differ
diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md
index d534c8cbe4b..672567a8d7d 100644
--- a/doc/user/project/repository/reducing_the_repo_size_using_git.md
+++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md
@@ -1,43 +1,105 @@
# Reducing the repository size using Git
A GitLab Enterprise Edition administrator can set a [repository size limit][admin-repo-size]
-which will prevent you to exceed it.
+which will prevent you from exceeding it.
When a project has reached its size limit, you will not be able to push to it,
create a new merge request, or merge existing ones. You will still be able to
create new issues, and clone the project though. Uploading LFS objects will
also be denied.
-In order to lift these restrictions, the administrator of the GitLab instance
-needs to increase the limit on the particular project that exceeded it or you
-need to instruct Git to rewrite changes.
-
If you exceed the repository size limit, your first thought might be to remove
-some data, make a new commit and push back to the repository. Unfortunately,
-it's not so easy and that workflow won't work. Deleting files in a commit doesn't
-actually reduce the size of the repo since the earlier commits and blobs are
-still around. What you need to do is rewrite history with Git's
-[`filter-branch` option][gitscm].
+some data, make a new commit and push back to the repository. Perhaps you can
+move some blobs to LFS, or remove some old dependency updates from history.
+Unfortunately, it's not so easy and that workflow won't work. Deleting files in
+a commit doesn't actually reduce the size of the repo since the earlier commits
+and blobs are still around. What you need to do is rewrite history with Git's
+[`filter-branch` option][gitscm], or a tool like the [BFG Repo-Cleaner][bfg].
Note that even with that method, until `git gc` runs on the GitLab side, the
-"removed" commits and blobs will still be around. And if a commit was ever
-included in an MR, or if a build was run for a commit, or if a user commented
-on it, it will be kept around too. So, in these cases the size will not decrease.
-
-The only fool proof way to actually decrease the repository size is to prune all
-the unneeded stuff locally, and then create a new project on GitLab and start
-using that instead.
+"removed" commits and blobs will still be around. You also need to be able to
+push the rewritten history to GitLab, which may be impossible if you've already
+exceeded the maximum size limit.
-With that being said, you can try reducing your repository size with the
-following method.
-
-## Using `git filter-branch` to purge files
+In order to lift these restrictions, the administrator of the GitLab instance
+needs to increase the limit on the particular project that exceeded it, so it's
+always better to spot that you're approaching the limit and act proactively to
+stay underneath it. If you hit the limit, and your admin can't - or won't -
+temporarily increase it for you, your only option is to prune all the unneeded
+stuff locally, and then create a new project on GitLab and start using that
+instead.
+
+If you can continue to use the original project, we recommend [using the
+BFG Repo-Cleaner](#using-the-bfg-repo-cleaner). It's faster and simpler than
+`git filter-branch`, and GitLab can use its account of what has changed to clean
+up its own internal state, maximizing the space saved.
> **Warning:**
> Make sure to first make a copy of your repository since rewriting history will
> purge the files and information you are about to delete. Also make sure to
> inform any collaborators to not use `pull` after your changes, but use `rebase`.
+> **Warning:**
+> This process is not suitable for removing sensitive data like password or keys
+> from your repository. Information about commits, including file content, is
+> cached in the database, and will remain visible even after they have been
+> removed from the repository.
+
+## Using the BFG Repo-Cleaner
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/19376) in GitLab 11.6.
+
+1. [Install BFG](https://rtyley.github.io/bfg-repo-cleaner/).
+
+1. Navigate to your repository:
+
+ ```
+ cd my_repository/
+ ```
+
+1. Change to the branch you want to remove the big file from:
+
+ ```
+ git checkout master
+ ```
+
+1. Create a commit removing the large file from the branch, if it still exists:
+
+ ```
+ git rm path/to/big_file.mpg
+ git commit -m 'Remove unneeded large file'
+ ```
+
+1. Rewrite history:
+
+ ```
+ bfg --delete-files path/to/big_file.mpg
+ ```
+
+ An object map file will be written to `object-id-map.old-new.txt`. Keep it
+ around - you'll need it for the final step!
+
+1. Force-push the changes to GitLab:
+
+ ```
+ git push --force-with-lease origin master
+ ```
+
+ If this step fails, someone has changed the `master` branch while you were
+ rewriting history. You could restore the branch and re-run BFG to preserve
+ their changes, or use `git push --force` to overwrite their changes.
+
+1. Navigate to **Project > Settings > Repository > Repository Cleanup**:
+
+ ![Repository settings cleanup form](img/repository_cleanup.png)
+
+ Upload the `object-id-map.old-new.txt` file and press **Start cleanup**.
+ This will remove any internal git references to the old commits, and run
+ `git gc` against the repository. You will receive an email once it has
+ completed.
+
+## Using `git filter-branch`
+
1. Navigate to your repository:
```
@@ -70,11 +132,6 @@ following method.
Your repository should now be below the size limit.
-> **Note:**
-> As an alternative to `filter-branch`, you can use the `bfg` tool with a
-> command like: `bfg --delete-files path/to/big_file.mpg`. Read the
-> [BFG Repo-Cleaner][bfg] documentation for more information.
-
[admin-repo-size]: https://docs.gitlab.com/ee/user/admin_area/settings/account_and_limit_settings.html#repository-size-limit
[bfg]: https://rtyley.github.io/bfg-repo-cleaner/
[gitscm]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#The-Nuclear-Option:-filter-branch
diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb
new file mode 100644
index 00000000000..2d1d8435cf3
--- /dev/null
+++ b/lib/gitlab/git/repository_cleaner.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Git
+ class RepositoryCleaner
+ include Gitlab::Git::WrapsGitalyErrors
+
+ attr_reader :repository
+
+ # 'repository' is a Gitlab::Git::Repository
+ def initialize(repository)
+ @repository = repository
+ end
+
+ def apply_bfg_object_map(io)
+ wrapped_gitaly_errors do
+ gitaly_cleanup_client.apply_bfg_object_map(io)
+ end
+ end
+
+ private
+
+ def gitaly_cleanup_client
+ @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
new file mode 100644
index 00000000000..8e412a9b3ef
--- /dev/null
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GitalyClient
+ class CleanupService
+ attr_reader :repository, :gitaly_repo, :storage
+
+ # 'repository' is a Gitlab::Git::Repository
+ def initialize(repository)
+ @repository = repository
+ @gitaly_repo = repository.gitaly_repository
+ @storage = repository.storage
+ end
+
+ def apply_bfg_object_map(io)
+ first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo)
+
+ enum = Enumerator.new do |y|
+ y.yield first_request
+
+ while data = io.read(RepositoryService::MAX_MSG_SIZE)
+ y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data)
+ end
+ end
+
+ GitalyClient.call(
+ storage,
+ :cleanup_service,
+ :apply_bfg_object_map,
+ enum,
+ timeout: GitalyClient.no_timeout
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 7cdea9d1ce4..d10d4f2f746 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -115,6 +115,7 @@ excluded_attributes:
- :remote_mirror_available_overridden
- :description_html
- :repository_languages
+ - :bfg_object_map
namespaces:
- :runners_token
- :runners_token_encrypted
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 23ee90ff0dd..2b9beea17ce 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -225,7 +225,7 @@ msgstr ""
msgid "2FA enabled"
msgstr ""
-msgid "403|Please contact your GitLab administrator to get the permission."
+msgid "403|Please contact your GitLab administrator to get permission."
msgstr ""
msgid "403|You don't have the permission to access this page."
@@ -1244,6 +1244,9 @@ msgstr ""
msgid "Choose a branch/tag (e.g. %{master}) or enter a commit (e.g. %{sha}) to see what's changed or to create a merge request."
msgstr ""
+msgid "Choose a file"
+msgstr ""
+
msgid "Choose a template..."
msgstr ""
@@ -2939,6 +2942,9 @@ msgstr ""
msgid "Failed to update issues, please try again."
msgstr ""
+msgid "Failed to upload object map file"
+msgstr ""
+
msgid "Failure"
msgstr ""
@@ -4333,6 +4339,9 @@ msgstr ""
msgid "No file chosen"
msgstr ""
+msgid "No file selected"
+msgstr ""
+
msgid "No files found."
msgstr ""
@@ -5489,6 +5498,12 @@ msgstr ""
msgid "Repository URL"
msgstr ""
+msgid "Repository cleanup"
+msgstr ""
+
+msgid "Repository cleanup has started. You will receive an email once the cleanup operation is complete."
+msgstr ""
+
msgid "Repository maintenance"
msgstr ""
@@ -6162,6 +6177,9 @@ msgstr ""
msgid "Start and due date"
msgstr ""
+msgid "Start cleanup"
+msgstr ""
+
msgid "Start date"
msgstr ""
@@ -6377,6 +6395,9 @@ msgstr ""
msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage."
msgstr ""
+msgid "The maximum file size allowed is %{max_attachment_size}mb"
+msgstr ""
+
msgid "The maximum file size allowed is 200KB."
msgstr ""
@@ -7059,6 +7080,9 @@ msgstr ""
msgid "Upload file"
msgstr ""
+msgid "Upload object map"
+msgstr ""
+
msgid "UploadLink|click to upload"
msgstr ""
diff --git a/spec/controllers/projects/settings/repository_controller_spec.rb b/spec/controllers/projects/settings/repository_controller_spec.rb
index 9cee40b7553..69ec971bb75 100644
--- a/spec/controllers/projects/settings/repository_controller_spec.rb
+++ b/spec/controllers/projects/settings/repository_controller_spec.rb
@@ -17,4 +17,35 @@ describe Projects::Settings::RepositoryController do
expect(response).to render_template(:show)
end
end
+
+ describe 'PUT cleanup' do
+ def do_put!
+ object_map = fixture_file_upload('spec/fixtures/bfg_object_map.txt')
+
+ Sidekiq::Testing.fake! do
+ put :cleanup, namespace_id: project.namespace, project_id: project, project: { object_map: object_map }
+ end
+ end
+
+ context 'feature enabled' do
+ it 'enqueues a RepositoryCleanupWorker' do
+ stub_feature_flags(project_cleanup: true)
+
+ do_put!
+
+ expect(response).to redirect_to project_settings_repository_path(project)
+ expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+ end
+ end
+
+ context 'feature disabled' do
+ it 'shows a 404 error' do
+ stub_feature_flags(project_cleanup: false)
+
+ do_put!
+
+ expect(response).to have_gitlab_http_status(404)
+ end
+ end
+ end
end
diff --git a/spec/features/projects/settings/repository_settings_spec.rb b/spec/features/projects/settings/repository_settings_spec.rb
index b7a22316d26..418e22f8c35 100644
--- a/spec/features/projects/settings/repository_settings_spec.rb
+++ b/spec/features/projects/settings/repository_settings_spec.rb
@@ -196,5 +196,40 @@ describe 'Projects > Settings > Repository settings' do
end
end
end
+
+ context 'repository cleanup settings' do
+ let(:object_map_file) { Rails.root.join('spec', 'fixtures', 'bfg_object_map.txt') }
+
+ context 'feature enabled' do
+ it 'uploads an object map file', :js do
+ stub_feature_flags(project_cleanup: true)
+
+ visit project_settings_repository_path(project)
+
+ expect(page).to have_content('Repository cleanup')
+
+ page.within('#cleanup') do
+ attach_file('project[bfg_object_map]', object_map_file, visible: false)
+
+ Sidekiq::Testing.fake! do
+ click_button 'Start cleanup'
+ end
+ end
+
+ expect(page).to have_content('Repository cleanup has started')
+ expect(RepositoryCleanupWorker.jobs.count).to eq(1)
+ end
+ end
+
+ context 'feature disabled' do
+ it 'does not show the settings' do
+ stub_feature_flags(project_cleanup: false)
+
+ visit project_settings_repository_path(project)
+
+ expect(page).not_to have_content('Repository cleanup')
+ end
+ end
+ end
end
end
diff --git a/spec/fixtures/bfg_object_map.txt b/spec/fixtures/bfg_object_map.txt
new file mode 100644
index 00000000000..c60171d8770
--- /dev/null
+++ b/spec/fixtures/bfg_object_map.txt
@@ -0,0 +1 @@
+f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 e242ed3bffccdf271b7fbaf34ed72d089537b42f
diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb
index 976b6c312b4..a857b7646b2 100644
--- a/spec/helpers/projects_helper_spec.rb
+++ b/spec/helpers/projects_helper_spec.rb
@@ -471,6 +471,31 @@ describe ProjectsHelper do
end
end
+ describe 'link_to_bfg' do
+ subject { helper.link_to_bfg }
+
+ it 'generates a hardcoded link to the BFG Repo-Cleaner' do
+ result = helper.link_to_bfg
+ doc = Nokogiri::HTML.fragment(result)
+
+ expect(doc.children.size).to eq(1)
+
+ link = doc.children.first
+
+ aggregate_failures do
+ expect(result).to be_html_safe
+
+ expect(link.name).to eq('a')
+ expect(link[:target]).to eq('_blank')
+ expect(link[:rel]).to eq('noopener noreferrer')
+ expect(link[:href]).to eq('https://rtyley.github.io/bfg-repo-cleaner/')
+ expect(link.inner_html).to eq('BFG')
+
+ expect(result).to be_html_safe
+ end
+ end
+ end
+
describe '#legacy_render_context' do
it 'returns the redcarpet engine' do
params = { legacy_render: '1' }
diff --git a/spec/javascripts/lib/utils/file_upload_spec.js b/spec/javascripts/lib/utils/file_upload_spec.js
new file mode 100644
index 00000000000..92c9cc70aaf
--- /dev/null
+++ b/spec/javascripts/lib/utils/file_upload_spec.js
@@ -0,0 +1,36 @@
+import fileUpload from '~/lib/utils/file_upload';
+
+describe('File upload', () => {
+ beforeEach(() => {
+ setFixtures(`
+ <form>
+ <button class="js-button" type="button">Click me!</button>
+ <input type="text" class="js-input" />
+ <span class="js-filename"></span>
+ </form>
+ `);
+
+ fileUpload('.js-button', '.js-input');
+ });
+
+ it('clicks file input after clicking button', () => {
+ const btn = document.querySelector('.js-button');
+ const input = document.querySelector('.js-input');
+
+ spyOn(input, 'click');
+
+ btn.click();
+
+ expect(input.click).toHaveBeenCalled();
+ });
+
+ it('updates file name text', () => {
+ const input = document.querySelector('.js-input');
+
+ input.value = 'path/to/file/index.js';
+
+ input.dispatchEvent(new CustomEvent('change'));
+
+ expect(document.querySelector('.js-filename').textContent).toEqual('index.js');
+ });
+});
diff --git a/spec/lib/gitlab/git/repository_cleaner_spec.rb b/spec/lib/gitlab/git/repository_cleaner_spec.rb
new file mode 100644
index 00000000000..a9d9e67ef94
--- /dev/null
+++ b/spec/lib/gitlab/git/repository_cleaner_spec.rb
@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Gitlab::Git::RepositoryCleaner do
+ let(:project) { create(:project, :repository) }
+ let(:repository) { project.repository }
+ let(:head_sha) { repository.head_commit.id }
+
+ let(:object_map) { StringIO.new("#{head_sha} #{'0' * 40}") }
+
+ subject(:cleaner) { described_class.new(repository.raw) }
+
+ describe '#apply_bfg_object_map' do
+ it 'removes internal references pointing at SHAs in the object map' do
+ # Create some refs we expect to be removed
+ repository.keep_around(head_sha)
+ repository.create_ref(head_sha, 'refs/environments/1')
+ repository.create_ref(head_sha, 'refs/merge-requests/1')
+ repository.create_ref(head_sha, 'refs/heads/_keep')
+ repository.create_ref(head_sha, 'refs/tags/_keep')
+
+ cleaner.apply_bfg_object_map(object_map)
+
+ aggregate_failures do
+ expect(repository.kept_around?(head_sha)).to be_falsy
+ expect(repository.ref_exists?('refs/environments/1')).to be_falsy
+ expect(repository.ref_exists?('refs/merge-requests/1')).to be_falsy
+ expect(repository.ref_exists?('refs/heads/_keep')).to be_truthy
+ expect(repository.ref_exists?('refs/tags/_keep')).to be_truthy
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb
new file mode 100644
index 00000000000..369deff732a
--- /dev/null
+++ b/spec/lib/gitlab/gitaly_client/cleanup_service_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Gitlab::GitalyClient::CleanupService do
+ let(:project) { create(:project) }
+ let(:storage_name) { project.repository_storage }
+ let(:relative_path) { project.disk_path + '.git' }
+ let(:client) { described_class.new(project.repository) }
+
+ describe '#apply_bfg_object_map' do
+ it 'sends an apply_bfg_object_map message' do
+ expect_any_instance_of(Gitaly::CleanupService::Stub)
+ .to receive(:apply_bfg_object_map)
+ .with(kind_of(Enumerator), kind_of(Hash))
+ .and_return(double)
+
+ client.apply_bfg_object_map(StringIO.new)
+ end
+ end
+end
diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb
index 2d8da7673dc..0f6c2604984 100644
--- a/spec/services/notification_service_spec.rb
+++ b/spec/services/notification_service_spec.rb
@@ -2146,6 +2146,27 @@ describe NotificationService, :mailer do
end
end
+ describe 'Repository cleanup' do
+ let(:user) { create(:user) }
+ let(:project) { create(:project) }
+
+ describe '#repository_cleanup_success' do
+ it 'emails the specified user only' do
+ notification.repository_cleanup_success(project, user)
+
+ should_email(user)
+ end
+ end
+
+ describe '#repository_cleanup_failure' do
+ it 'emails the specified user only' do
+ notification.repository_cleanup_failure(project, user, 'Some error')
+
+ should_email(user)
+ end
+ end
+ end
+
def build_team(project)
@u_watcher = create_global_setting_for(create(:user), :watch)
@u_participating = create_global_setting_for(create(:user), :participating)
diff --git a/spec/services/projects/cleanup_service_spec.rb b/spec/services/projects/cleanup_service_spec.rb
new file mode 100644
index 00000000000..3d4587ce2a1
--- /dev/null
+++ b/spec/services/projects/cleanup_service_spec.rb
@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+describe Projects::CleanupService do
+ let(:project) { create(:project, :repository, bfg_object_map: fixture_file_upload('spec/fixtures/bfg_object_map.txt')) }
+ let(:object_map) { project.bfg_object_map }
+
+ subject(:service) { described_class.new(project) }
+
+ describe '#execute' do
+ it 'runs the apply_bfg_object_map gitaly RPC' do
+ expect_next_instance_of(Gitlab::Git::RepositoryCleaner) do |cleaner|
+ expect(cleaner).to receive(:apply_bfg_object_map).with(kind_of(IO))
+ end
+
+ service.execute
+ end
+
+ it 'runs garbage collection on the repository' do
+ expect_next_instance_of(GitGarbageCollectWorker) do |worker|
+ expect(worker).to receive(:perform)
+ end
+
+ service.execute
+ end
+
+ it 'clears the repository cache' do
+ expect(project.repository).to receive(:expire_all_method_caches)
+
+ service.execute
+ end
+
+ it 'removes the object map file' do
+ service.execute
+
+ expect(object_map.exists?).to be_falsy
+ end
+
+ it 'raises an error if no object map can be found' do
+ object_map.remove!
+
+ expect { service.execute }.to raise_error(described_class::NoUploadError)
+ end
+ end
+end
diff --git a/spec/workers/repository_cleanup_worker_spec.rb b/spec/workers/repository_cleanup_worker_spec.rb
new file mode 100644
index 00000000000..3adae0b6cfa
--- /dev/null
+++ b/spec/workers/repository_cleanup_worker_spec.rb
@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe RepositoryCleanupWorker do
+ let(:project) { create(:project) }
+ let(:user) { create(:user) }
+
+ subject(:worker) { described_class.new }
+
+ describe '#perform' do
+ it 'executes the cleanup service and sends a success notification' do
+ expect_next_instance_of(Projects::CleanupService) do |service|
+ expect(service.project).to eq(project)
+ expect(service.current_user).to eq(user)
+
+ expect(service).to receive(:execute)
+ end
+
+ expect_next_instance_of(NotificationService) do |service|
+ expect(service).to receive(:repository_cleanup_success).with(project, user)
+ end
+
+ worker.perform(project.id, user.id)
+ end
+
+ it 'raises an error if the project cannot be found' do
+ project.destroy
+
+ expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+ end
+
+ it 'raises an error if the user cannot be found' do
+ user.destroy
+
+ expect { worker.perform(project.id, user.id) }.to raise_error(ActiveRecord::RecordNotFound)
+ end
+ end
+
+ describe '#sidekiq_retries_exhausted' do
+ let(:job) { { 'args' => [project.id, user.id], 'error_message' => 'Error' } }
+
+ it 'does not send a failure notification for a RecordNotFound error' do
+ expect(NotificationService).not_to receive(:new)
+
+ described_class.sidekiq_retries_exhausted_block.call(job, ActiveRecord::RecordNotFound.new)
+ end
+
+ it 'sends a failure notification' do
+ expect_next_instance_of(NotificationService) do |service|
+ expect(service).to receive(:repository_cleanup_failure).with(project, user, 'Error')
+ end
+
+ described_class.sidekiq_retries_exhausted_block.call(job, StandardError.new)
+ end
+ end
+end