From 9d306eb132bf153a0c93dd870d3a098028f12384 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 3 May 2016 12:41:23 +0200 Subject: picking stuff from ui related to import --- app/workers/project_impor_worker.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 app/workers/project_impor_worker.rb (limited to 'app/workers') diff --git a/app/workers/project_impor_worker.rb b/app/workers/project_impor_worker.rb new file mode 100644 index 00000000000..0ef353fa441 --- /dev/null +++ b/app/workers/project_impor_worker.rb @@ -0,0 +1,25 @@ +class ProjectImportWorker + include Sidekiq::Worker + include Gitlab::ShellAdapter + + sidekiq_options queue: :gitlab_shell, retry: false + + def perform(current_user_id, tmp_file, namespace_id, path) + current_user = User.find(current_user_id) + + project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, + owner: current_user, + namespace_id: namespace_id, + project_path: path) + + # TODO: Move this to import service + # if result[:status] == :error + # project.update(import_error: result[:message]) + # project.import_fail + # return + # end + + project.repository.after_import + project.import_finish + end +end \ No newline at end of file -- cgit v1.2.1 From 6139e1433c7f1925a7fc599106656990410a4184 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 3 May 2016 12:46:00 +0200 Subject: fixing some rubucop warnings --- app/workers/project_impor_worker.rb | 25 ------------------------- app/workers/project_import_worker.rb | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 app/workers/project_impor_worker.rb create mode 100644 app/workers/project_import_worker.rb (limited to 'app/workers') diff --git a/app/workers/project_impor_worker.rb b/app/workers/project_impor_worker.rb deleted file mode 100644 index 0ef353fa441..00000000000 --- a/app/workers/project_impor_worker.rb +++ /dev/null @@ -1,25 +0,0 @@ -class ProjectImportWorker - include Sidekiq::Worker - include Gitlab::ShellAdapter - - sidekiq_options queue: :gitlab_shell, retry: false - - def perform(current_user_id, tmp_file, namespace_id, path) - current_user = User.find(current_user_id) - - project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, - owner: current_user, - namespace_id: namespace_id, - project_path: path) - - # TODO: Move this to import service - # if result[:status] == :error - # project.update(import_error: result[:message]) - # project.import_fail - # return - # end - - project.repository.after_import - project.import_finish - end -end \ No newline at end of file diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb new file mode 100644 index 00000000000..0e8b9552442 --- /dev/null +++ b/app/workers/project_import_worker.rb @@ -0,0 +1,25 @@ +class ProjectImportWorker + include Sidekiq::Worker + include Gitlab::ShellAdapter + + sidekiq_options queue: :gitlab_shell, retry: false + + def perform(current_user_id, tmp_file, namespace_id, path) + current_user = User.find(current_user_id) + + project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, + owner: current_user, + namespace_id: namespace_id, + project_path: path) + + # TODO: Move this to import service + # if result[:status] == :error + # project.update(import_error: result[:message]) + # project.import_fail + # return + # end + + project.repository.after_import + project.import_finish + end +end -- cgit v1.2.1 From bc8eebf04a32d893d5b536deb2e7ffc205d9fcda Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 4 May 2016 10:59:33 +0200 Subject: add export worker to process project export async --- app/workers/project_export_worker.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 app/workers/project_export_worker.rb (limited to 'app/workers') diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb new file mode 100644 index 00000000000..a1add5395b7 --- /dev/null +++ b/app/workers/project_export_worker.rb @@ -0,0 +1,13 @@ +class ProjectExportWorker + include Sidekiq::Worker + + # TODO: enabled retry - disabled for QA purposes + sidekiq_options queue: :gitlab_shell, retry: false + + def perform(current_user_id, project_id) + current_user = User.find(current_user_id) + project = Project.find(project_id) + ::Projects::ImportExport::ExportService.new(project, current_user).execute + # TODO : Handle errors + end +end -- cgit v1.2.1 From 9b21207305f7516b132211b5ead584beb37b8948 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 4 May 2016 11:01:55 +0200 Subject: updated import worker --- app/workers/project_import_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb index 0e8b9552442..aaa59f79d7f 100644 --- a/app/workers/project_import_worker.rb +++ b/app/workers/project_import_worker.rb @@ -1,7 +1,7 @@ class ProjectImportWorker include Sidekiq::Worker - include Gitlab::ShellAdapter + # TODO: enabled retry - disabled for QA purposes sidekiq_options queue: :gitlab_shell, retry: false def perform(current_user_id, tmp_file, namespace_id, path) -- cgit v1.2.1 From 9f5dd2de4e85eba70cd469b36db1c19988c3603f Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 5 May 2016 18:12:24 +0200 Subject: handling errors a bit better on import failure --- app/workers/project_import_worker.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb index aaa59f79d7f..cc75da704f6 100644 --- a/app/workers/project_import_worker.rb +++ b/app/workers/project_import_worker.rb @@ -11,15 +11,15 @@ class ProjectImportWorker owner: current_user, namespace_id: namespace_id, project_path: path) + if project + project.repository.after_import + project.import_finish + else + logger.error("There was an error during the import: #{tmpfile}") + end + end - # TODO: Move this to import service - # if result[:status] == :error - # project.update(import_error: result[:message]) - # project.import_fail - # return - # end - - project.repository.after_import - project.import_finish + def logger + Sidekiq.logger end end -- cgit v1.2.1 From 5086e899fe26447e6e24064c9a05d3989c1e8448 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 13:21:00 +0200 Subject: fix log statement --- app/workers/project_import_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb index cc75da704f6..c6a6d24b4e5 100644 --- a/app/workers/project_import_worker.rb +++ b/app/workers/project_import_worker.rb @@ -15,7 +15,7 @@ class ProjectImportWorker project.repository.after_import project.import_finish else - logger.error("There was an error during the import: #{tmpfile}") + logger.error("There was an error during the import: #{tmp_file}") end end -- cgit v1.2.1 From cffae0d22ee159b78f66eb68cae9df4bcb9e83e5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 14:51:25 +0200 Subject: fixing more export problems --- app/workers/project_export_worker.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'app/workers') diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index a1add5395b7..3616b37d2ad 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,13 +1,12 @@ class ProjectExportWorker include Sidekiq::Worker - # TODO: enabled retry - disabled for QA purposes + # TODO: enable retry - disabled for QA purposes sidekiq_options queue: :gitlab_shell, retry: false def perform(current_user_id, project_id) current_user = User.find(current_user_id) project = Project.find(project_id) ::Projects::ImportExport::ExportService.new(project, current_user).execute - # TODO : Handle errors end end -- cgit v1.2.1 From a61456e44e8f26067187643a3f1b74b484428bad Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 11 May 2016 17:22:45 +0200 Subject: refactored import to use shared error stuff and fixed a few issues with recent refactorings --- app/workers/project_import_worker.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb index c6a6d24b4e5..b2902c3278e 100644 --- a/app/workers/project_import_worker.rb +++ b/app/workers/project_import_worker.rb @@ -13,7 +13,6 @@ class ProjectImportWorker project_path: path) if project project.repository.after_import - project.import_finish else logger.error("There was an error during the import: #{tmp_file}") end -- cgit v1.2.1 From 3e99123095b26988de67a94b0e7a5207c1ef5ae2 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 10:57:46 +0200 Subject: Fix merge conflicts - squashed commit # Conflicts: # app/models/project.rb --- app/workers/emails_on_push_worker.rb | 44 ++++++++++++++++++++++++--------- app/workers/repository_fork_worker.rb | 6 ++--- app/workers/repository_import_worker.rb | 3 +-- 3 files changed, 36 insertions(+), 17 deletions(-) (limited to 'app/workers') diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index c4d8595d45d..971f969e25e 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -1,6 +1,9 @@ class EmailsOnPushWorker include Sidekiq::Worker + sidekiq_options queue: :mailers + attr_reader :email, :skip_premailer + def perform(project_id, recipients, push_data, options = {}) options.symbolize_keys! options.reverse_merge!( @@ -25,15 +28,18 @@ class EmailsOnPushWorker :push end + diff_refs = nil compare = nil reverse_compare = false if action == :push compare = Gitlab::Git::Compare.new(project.repository.raw_repository, before_sha, after_sha) + diff_refs = [project.merge_base_commit(before_sha, after_sha), project.commit(after_sha)] return false if compare.same if compare.commits.empty? compare = Gitlab::Git::Compare.new(project.repository.raw_repository, after_sha, before_sha) + diff_refs = [project.merge_base_commit(after_sha, before_sha), project.commit(before_sha)] reverse_compare = true @@ -41,26 +47,42 @@ class EmailsOnPushWorker end end - recipients.split(" ").each do |recipient| + recipients.split.each do |recipient| begin - Notify.repository_push_email( - project_id, + send_email( recipient, - author_id: author_id, - ref: ref, - action: action, - compare: compare, - reverse_compare: reverse_compare, - send_from_committer_email: send_from_committer_email, - disable_diffs: disable_diffs - ).deliver_now + project_id, + author_id: author_id, + ref: ref, + action: action, + compare: compare, + reverse_compare: reverse_compare, + diff_refs: diff_refs, + send_from_committer_email: send_from_committer_email, + disable_diffs: disable_diffs + ) + # These are input errors and won't be corrected even if Sidekiq retries rescue Net::SMTPFatalError, Net::SMTPSyntaxError => e logger.info("Failed to send e-mail for project '#{project.name_with_namespace}' to #{recipient}: #{e}") end end ensure + @email = nil compare = nil GC.start end + + private + + def send_email(recipient, project_id, options) + # Generating the body of this email can be expensive, so only do it once + @skip_premailer ||= email.present? + @email ||= Notify.repository_push_email(project_id, options) + + email.to = recipient + email.add_message_id + email.header[:skip_premailer] = true if skip_premailer + email.deliver_now + end end diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index f9e32337983..d947f105516 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -15,8 +15,7 @@ class RepositoryForkWorker result = gitlab_shell.fork_repository(source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") - project.update(import_error: "The project could not be forked.") - project.import_fail + project.mark_import_as_failed('The project could not be forked.') return end @@ -24,8 +23,7 @@ class RepositoryForkWorker unless project.valid_repo? logger.error("Project #{project_id} had an invalid repository after fork") - project.update(import_error: "The forked repository is invalid.") - project.import_fail + project.mark_import_as_failed('The forked repository is invalid.') return end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 2937493c614..7d819fe78f8 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -13,8 +13,7 @@ class RepositoryImportWorker result = Projects::ImportService.new(project, current_user).execute if result[:status] == :error - project.update(import_error: result[:message]) - project.import_fail + project.mark_import_as_failed(result[:message]) return end -- cgit v1.2.1 From 721014c92799219d357b1b7c971d4c0b6050ff2a Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 3 Jun 2016 11:10:17 +0200 Subject: Revert "Fix merge conflicts - squashed commit" This reverts commit 3e99123095b26988de67a94b0e7a5207c1ef5ae2. --- app/workers/emails_on_push_worker.rb | 44 +++++++++------------------------ app/workers/repository_fork_worker.rb | 6 +++-- app/workers/repository_import_worker.rb | 3 ++- 3 files changed, 17 insertions(+), 36 deletions(-) (limited to 'app/workers') diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 971f969e25e..c4d8595d45d 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -1,9 +1,6 @@ class EmailsOnPushWorker include Sidekiq::Worker - sidekiq_options queue: :mailers - attr_reader :email, :skip_premailer - def perform(project_id, recipients, push_data, options = {}) options.symbolize_keys! options.reverse_merge!( @@ -28,18 +25,15 @@ class EmailsOnPushWorker :push end - diff_refs = nil compare = nil reverse_compare = false if action == :push compare = Gitlab::Git::Compare.new(project.repository.raw_repository, before_sha, after_sha) - diff_refs = [project.merge_base_commit(before_sha, after_sha), project.commit(after_sha)] return false if compare.same if compare.commits.empty? compare = Gitlab::Git::Compare.new(project.repository.raw_repository, after_sha, before_sha) - diff_refs = [project.merge_base_commit(after_sha, before_sha), project.commit(before_sha)] reverse_compare = true @@ -47,42 +41,26 @@ class EmailsOnPushWorker end end - recipients.split.each do |recipient| + recipients.split(" ").each do |recipient| begin - send_email( - recipient, + Notify.repository_push_email( project_id, - author_id: author_id, - ref: ref, - action: action, - compare: compare, - reverse_compare: reverse_compare, - diff_refs: diff_refs, - send_from_committer_email: send_from_committer_email, - disable_diffs: disable_diffs - ) - + recipient, + author_id: author_id, + ref: ref, + action: action, + compare: compare, + reverse_compare: reverse_compare, + send_from_committer_email: send_from_committer_email, + disable_diffs: disable_diffs + ).deliver_now # These are input errors and won't be corrected even if Sidekiq retries rescue Net::SMTPFatalError, Net::SMTPSyntaxError => e logger.info("Failed to send e-mail for project '#{project.name_with_namespace}' to #{recipient}: #{e}") end end ensure - @email = nil compare = nil GC.start end - - private - - def send_email(recipient, project_id, options) - # Generating the body of this email can be expensive, so only do it once - @skip_premailer ||= email.present? - @email ||= Notify.repository_push_email(project_id, options) - - email.to = recipient - email.add_message_id - email.header[:skip_premailer] = true if skip_premailer - email.deliver_now - end end diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d947f105516..f9e32337983 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -15,7 +15,8 @@ class RepositoryForkWorker result = gitlab_shell.fork_repository(source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") - project.mark_import_as_failed('The project could not be forked.') + project.update(import_error: "The project could not be forked.") + project.import_fail return end @@ -23,7 +24,8 @@ class RepositoryForkWorker unless project.valid_repo? logger.error("Project #{project_id} had an invalid repository after fork") - project.mark_import_as_failed('The forked repository is invalid.') + project.update(import_error: "The forked repository is invalid.") + project.import_fail return end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 7d819fe78f8..2937493c614 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -13,7 +13,8 @@ class RepositoryImportWorker result = Projects::ImportService.new(project, current_user).execute if result[:status] == :error - project.mark_import_as_failed(result[:message]) + project.update(import_error: result[:message]) + project.import_fail return end -- cgit v1.2.1 From 3f7ed550110daaec8a76af7146b701dfc0210e60 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 12:47:07 +0200 Subject: lots of refactoring to reuse import service --- app/workers/project_import_worker.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb index b2902c3278e..b18d453702e 100644 --- a/app/workers/project_import_worker.rb +++ b/app/workers/project_import_worker.rb @@ -7,10 +7,10 @@ class ProjectImportWorker def perform(current_user_id, tmp_file, namespace_id, path) current_user = User.find(current_user_id) - project = Gitlab::ImportExport::ImportService.execute(archive_file: tmp_file, - owner: current_user, - namespace_id: namespace_id, - project_path: path) + project = Gitlab::ImportExport::Importer.execute(archive_file: tmp_file, + owner: current_user, + namespace_id: namespace_id, + project_path: path) if project project.repository.after_import else -- cgit v1.2.1 From 9ecebaaea16d206ed20a2f4fc0021a2145c873f5 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 16:31:03 +0200 Subject: adding notifications stuff and more refactoring for exporting projects --- app/workers/project_export_worker.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/workers') diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 3616b37d2ad..d2dcdb46219 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -7,6 +7,7 @@ class ProjectExportWorker def perform(current_user_id, project_id) current_user = User.find(current_user_id) project = Project.find(project_id) + ::Projects::ImportExport::ExportService.new(project, current_user).execute end end -- cgit v1.2.1 From dee6fba5c9df60be370b881f074fac13d16b83ed Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 21:42:49 +0200 Subject: get rid of old worker --- app/workers/project_import_worker.rb | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 app/workers/project_import_worker.rb (limited to 'app/workers') diff --git a/app/workers/project_import_worker.rb b/app/workers/project_import_worker.rb deleted file mode 100644 index b18d453702e..00000000000 --- a/app/workers/project_import_worker.rb +++ /dev/null @@ -1,24 +0,0 @@ -class ProjectImportWorker - include Sidekiq::Worker - - # TODO: enabled retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false - - def perform(current_user_id, tmp_file, namespace_id, path) - current_user = User.find(current_user_id) - - project = Gitlab::ImportExport::Importer.execute(archive_file: tmp_file, - owner: current_user, - namespace_id: namespace_id, - project_path: path) - if project - project.repository.after_import - else - logger.error("There was an error during the import: #{tmp_file}") - end - end - - def logger - Sidekiq.logger - end -end -- cgit v1.2.1 From 36ccaca35a7be19498d779a5b781cc4e3f0dede8 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Tue, 14 Jun 2016 22:11:21 +0200 Subject: project export archiver --- app/workers/gitlab_project_archive_worker.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 app/workers/gitlab_project_archive_worker.rb (limited to 'app/workers') diff --git a/app/workers/gitlab_project_archive_worker.rb b/app/workers/gitlab_project_archive_worker.rb new file mode 100644 index 00000000000..63e7086d41a --- /dev/null +++ b/app/workers/gitlab_project_archive_worker.rb @@ -0,0 +1,9 @@ +class GitlabProjectArchiveWorker + include Sidekiq::Worker + + sidekiq_options queue: :default + + def perform + Project.archive_gitlab_exports! + end +end -- cgit v1.2.1 From bacd2c4fa0253f7b7206dc949e5c2d8d132a8087 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 15 Jun 2016 09:03:54 +0200 Subject: some refactoring - renaming things, etc.. --- app/workers/gitlab_project_archive_worker.rb | 9 --------- app/workers/gitlab_remove_project_export_worker.rb | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 app/workers/gitlab_project_archive_worker.rb create mode 100644 app/workers/gitlab_remove_project_export_worker.rb (limited to 'app/workers') diff --git a/app/workers/gitlab_project_archive_worker.rb b/app/workers/gitlab_project_archive_worker.rb deleted file mode 100644 index 63e7086d41a..00000000000 --- a/app/workers/gitlab_project_archive_worker.rb +++ /dev/null @@ -1,9 +0,0 @@ -class GitlabProjectArchiveWorker - include Sidekiq::Worker - - sidekiq_options queue: :default - - def perform - Project.archive_gitlab_exports! - end -end diff --git a/app/workers/gitlab_remove_project_export_worker.rb b/app/workers/gitlab_remove_project_export_worker.rb new file mode 100644 index 00000000000..1d91897d520 --- /dev/null +++ b/app/workers/gitlab_remove_project_export_worker.rb @@ -0,0 +1,9 @@ +class GitlabRemoveProjectExportWorker + include Sidekiq::Worker + + sidekiq_options queue: :default + + def perform + Project.remove_gitlab_exports! + end +end -- cgit v1.2.1 From 8922712693a57d6c65620d03599fe7c4b4f84119 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 15 Jun 2016 09:06:10 +0200 Subject: enable retry on worker --- app/workers/project_export_worker.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'app/workers') diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index d2dcdb46219..39f6037e077 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -1,8 +1,7 @@ class ProjectExportWorker include Sidekiq::Worker - # TODO: enable retry - disabled for QA purposes - sidekiq_options queue: :gitlab_shell, retry: false + sidekiq_options queue: :gitlab_shell, retry: true def perform(current_user_id, project_id) current_user = User.find(current_user_id) -- cgit v1.2.1 From d9f6d5dd59281a8efcd6cb7b392b1ef3bafa778d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 15 Jun 2016 19:11:25 +0200 Subject: Do not check repos without push events --- app/workers/repository_check/single_repository_worker.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/workers') diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index f2d12ba5a7d..98ddf5d0688 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -15,7 +15,7 @@ module RepositoryCheck private def check(project) - if !git_fsck(project.repository) + if has_pushes?(project) && !git_fsck(project.repository) false elsif project.wiki_enabled? # Historically some projects never had their wiki repos initialized; @@ -44,5 +44,9 @@ module RepositoryCheck false end end + + def has_pushes?(project) + Project.with_push.exists?(project.id) + end end end -- cgit v1.2.1 From 13e37a3ee5c943525a99481b855d654e97e8597c Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 12:12:27 +0200 Subject: squashed merge and fixed conflicts --- app/workers/expire_build_artifacts_worker.rb | 13 +++++++++++++ app/workers/stuck_ci_builds_worker.rb | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 app/workers/expire_build_artifacts_worker.rb (limited to 'app/workers') diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb new file mode 100644 index 00000000000..c64ea108d52 --- /dev/null +++ b/app/workers/expire_build_artifacts_worker.rb @@ -0,0 +1,13 @@ +class ExpireBuildArtifactsWorker + include Sidekiq::Worker + + def perform + Rails.logger.info 'Cleaning old build artifacts' + + builds = Ci::Build.with_expired_artifacts + builds.find_each(batch_size: 50).each do |build| + Rails.logger.debug "Removing artifacts build #{build.id}..." + build.erase_artifacts! + end + end +end diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb index ca594e77e7c..6828013b377 100644 --- a/app/workers/stuck_ci_builds_worker.rb +++ b/app/workers/stuck_ci_builds_worker.rb @@ -6,7 +6,7 @@ class StuckCiBuildsWorker def perform Rails.logger.info 'Cleaning stuck builds' - builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago) + builds = Ci::Build.joins(:project).running_or_pending.where('ci_builds.updated_at < ?', BUILD_STUCK_TIMEOUT.ago) builds.find_each(batch_size: 50).each do |build| Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}" build.drop -- cgit v1.2.1 From 452c076a34cc11cc97f4b1c3113e86ce4367e055 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 16 Jun 2016 12:59:07 +0200 Subject: Revert "squashed merge and fixed conflicts" This reverts commit 13e37a3ee5c943525a99481b855d654e97e8597c. --- app/workers/expire_build_artifacts_worker.rb | 13 ------------- app/workers/stuck_ci_builds_worker.rb | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 app/workers/expire_build_artifacts_worker.rb (limited to 'app/workers') diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb deleted file mode 100644 index c64ea108d52..00000000000 --- a/app/workers/expire_build_artifacts_worker.rb +++ /dev/null @@ -1,13 +0,0 @@ -class ExpireBuildArtifactsWorker - include Sidekiq::Worker - - def perform - Rails.logger.info 'Cleaning old build artifacts' - - builds = Ci::Build.with_expired_artifacts - builds.find_each(batch_size: 50).each do |build| - Rails.logger.debug "Removing artifacts build #{build.id}..." - build.erase_artifacts! - end - end -end diff --git a/app/workers/stuck_ci_builds_worker.rb b/app/workers/stuck_ci_builds_worker.rb index 6828013b377..ca594e77e7c 100644 --- a/app/workers/stuck_ci_builds_worker.rb +++ b/app/workers/stuck_ci_builds_worker.rb @@ -6,7 +6,7 @@ class StuckCiBuildsWorker def perform Rails.logger.info 'Cleaning stuck builds' - builds = Ci::Build.joins(:project).running_or_pending.where('ci_builds.updated_at < ?', BUILD_STUCK_TIMEOUT.ago) + builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago) builds.find_each(batch_size: 50).each do |build| Rails.logger.debug "Dropping stuck #{build.status} build #{build.id} for runner #{build.runner_id}" build.drop -- cgit v1.2.1 From 86359ec854314574dccea75247f45590262b05c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Wed, 22 Jun 2016 17:04:51 -0400 Subject: Refactor repository paths handling to allow multiple git mount points --- app/workers/post_receive.rb | 6 +++--- app/workers/repository_fork_worker.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'app/workers') diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index f3327ca9e61..09035a7cf2d 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -4,10 +4,10 @@ class PostReceive sidekiq_options queue: :post_receive def perform(repo_path, identifier, changes) - if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s) - repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "") + if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) } + repo_path.gsub!(path[1].to_s, "") else - log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"") + log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"") end post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d947f105516..f7604e48f83 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -12,7 +12,7 @@ class RepositoryForkWorker return end - result = gitlab_shell.fork_repository(source_path, target_path) + result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") project.mark_import_as_failed('The project could not be forked.') -- cgit v1.2.1 From a9fa45f09e6b6188691f37d75883b22edce7bba1 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 20 Jun 2016 18:51:48 +0200 Subject: Represent DiffRefs as proper class instead of tuple array --- app/workers/emails_on_push_worker.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'app/workers') diff --git a/app/workers/emails_on_push_worker.rb b/app/workers/emails_on_push_worker.rb index 971f969e25e..8551288e2f2 100644 --- a/app/workers/emails_on_push_worker.rb +++ b/app/workers/emails_on_push_worker.rb @@ -28,18 +28,30 @@ class EmailsOnPushWorker :push end + merge_base_sha = project.merge_base_commit(before_sha, after_sha).try(:sha) + diff_refs = nil compare = nil reverse_compare = false if action == :push compare = Gitlab::Git::Compare.new(project.repository.raw_repository, before_sha, after_sha) - diff_refs = [project.merge_base_commit(before_sha, after_sha), project.commit(after_sha)] + + diff_refs = Gitlab::Diff::DiffRefs.new( + base_sha: merge_base_sha, + start_sha: before_sha, + head_sha: after_sha + ) return false if compare.same if compare.commits.empty? compare = Gitlab::Git::Compare.new(project.repository.raw_repository, after_sha, before_sha) - diff_refs = [project.merge_base_commit(after_sha, before_sha), project.commit(before_sha)] + + diff_refs = Gitlab::Diff::DiffRefs.new( + base_sha: merge_base_sha, + start_sha: after_sha, + head_sha: before_sha + ) reverse_compare = true -- cgit v1.2.1 From 3dc6bf2b71f995a3b6ca40ebbf9abb5c11397b8b Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 8 Jul 2016 12:32:58 -0700 Subject: Expire the branch cache after `git gc` runs Due to a stale NFS cache, it's possible that a branch lookup fails while `git gc` is running and causes missing branches in merge requests. Possible workaround for #15392 --- app/workers/git_garbage_collect_worker.rb | 14 ++++++++++++++ app/workers/gitlab_shell_one_shot_worker.rb | 10 ---------- 2 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 app/workers/git_garbage_collect_worker.rb delete mode 100644 app/workers/gitlab_shell_one_shot_worker.rb (limited to 'app/workers') diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb new file mode 100644 index 00000000000..2fa3c838f55 --- /dev/null +++ b/app/workers/git_garbage_collect_worker.rb @@ -0,0 +1,14 @@ +class GitGarbageCollectWorker + include Sidekiq::Worker + include Gitlab::ShellAdapter + + sidekiq_options queue: :gitlab_shell, retry: false + + def perform(project_id) + project = Project.find(project_id) + + gitlab_shell.gc(project.repository_storage_path, project.path_with_namespace) + # Expire the branch cache in case garbage collection caused a ref lookup to fail + project.repository.after_create_branch + end +end diff --git a/app/workers/gitlab_shell_one_shot_worker.rb b/app/workers/gitlab_shell_one_shot_worker.rb deleted file mode 100644 index 4ddbcf574d5..00000000000 --- a/app/workers/gitlab_shell_one_shot_worker.rb +++ /dev/null @@ -1,10 +0,0 @@ -class GitlabShellOneShotWorker - include Sidekiq::Worker - include Gitlab::ShellAdapter - - sidekiq_options queue: :gitlab_shell, retry: false - - def perform(action, *arg) - gitlab_shell.send(action, *arg) - end -end -- cgit v1.2.1