diff options
23 files changed, 112 insertions, 112 deletions
diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb index b8c276fb1bb..01257a68616 100644 --- a/app/controllers/admin/projects_controller.rb +++ b/app/controllers/admin/projects_controller.rb @@ -1,5 +1,5 @@ class Admin::ProjectsController < Admin::ApplicationController - before_action :project, only: [:show, :transfer, :repo_check] + before_action :project, only: [:show, :transfer, :repository_check] before_action :group, only: [:show, :transfer] def index @@ -8,7 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController @projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present? @projects = @projects.with_push if params[:with_push].present? @projects = @projects.abandoned if params[:abandoned].present? - @projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present? + @projects = @projects.where(last_repository_check_failed: true) if params[:last_repository_check_failed].present? @projects = @projects.non_archived unless params[:with_archived].present? @projects = @projects.search(params[:name]) if params[:name].present? @projects = @projects.sort(@sort = params[:sort]) @@ -31,24 +31,24 @@ class Admin::ProjectsController < Admin::ApplicationController redirect_to admin_namespace_project_path(@project.namespace, @project) end - def repo_check - SingleRepoCheckWorker.perform_async(@project.id) + def repository_check + SingleRepositoryCheckWorker.perform_async(@project.id) redirect_to( admin_namespace_project_path(@project.namespace, @project), - notice: 'Repo check was triggered' + notice: 'Repository check was triggered' ) end - def clear_repo_check_states + def clear_repository_check_states Project.update_all( - last_repo_check_failed: false, - last_repo_check_at: nil + last_repository_check_failed: false, + last_repository_check_at: nil ) redirect_to( admin_namespaces_projects_path, - notice: 'All project repo check states were cleared' + notice: 'All project states were cleared' ) end diff --git a/app/mailers/repo_check_mailer.rb b/app/mailers/repository_check_mailer.rb index d98533f120d..994054c8769 100644 --- a/app/mailers/repo_check_mailer.rb +++ b/app/mailers/repository_check_mailer.rb @@ -1,4 +1,4 @@ -class RepoCheckMailer < BaseMailer +class RepositoryCheckMailer < BaseMailer include ActionView::Helpers::TextHelper def notify(failed_count) diff --git a/app/views/admin/logs/show.html.haml b/app/views/admin/logs/show.html.haml index abcc93f4046..4b475a4d8fa 100644 --- a/app/views/admin/logs/show.html.haml +++ b/app/views/admin/logs/show.html.haml @@ -1,7 +1,7 @@ - page_title "Logs" - loggers = [Gitlab::GitLogger, Gitlab::AppLogger, Gitlab::ProductionLogger, Gitlab::SidekiqLogger, - Gitlab::RepoCheckLogger] + Gitlab::RepositoryCheckLogger] %ul.nav-links.log-tabs - loggers.each do |klass| %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml index ed360f2f012..c2bf0659841 100644 --- a/app/views/admin/projects/index.html.haml +++ b/app/views/admin/projects/index.html.haml @@ -41,19 +41,19 @@ %fieldset %strong Problems .checkbox - = label_tag :last_repo_check_failed do - = check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed] - %span Last repo check failed + = label_tag :last_repository_check_failed do + = check_box_tag :last_repository_check_failed, 1, params[:last_repository_check_failed] + %span Last repository check failed = hidden_field_tag :sort, params[:sort] = button_tag "Search", class: "btn submit btn-primary" = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel" - .panel.panel-default.repo-check-states + .panel.panel-default.repository-check-states .panel-heading - Repo check states + Repository check states .panel-body - = link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove" + = link_to 'Clear all', clear_repository_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove" %section.col-md-9 .panel.panel-default .panel-heading diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index a7a3f6349ef..5bef8e3ad57 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -5,12 +5,12 @@ %i.fa.fa-pencil-square-o Edit %hr -- if @project.last_repo_check_failed? +- if @project.last_repository_check_failed? .row .col-md-12 .panel .panel-heading.alert.alert-danger - Last repo check failed. See + Last repository check failed. See = link_to 'repocheck.log', admin_logs_path for error messages. .row @@ -103,19 +103,19 @@ .col-sm-offset-2.col-sm-10 = f.submit 'Transfer', class: 'btn btn-primary' - .panel.panel-default.repo-check + .panel.panel-default.repository-check .panel-heading - Repo check + Repository check .panel-body - = form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f| + = form_for @project, url: repository_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f| .form-group - - if @project.last_repo_check_at.nil? + - if @project.last_repository_check_at.nil? This repository has never been checked. - else This repository was last checked - = @project.last_repo_check_at.to_s(:medium) + '.' + = @project.last_repository_check_at.to_s(:medium) + '.' The check - - if @project.last_repo_check_failed? + - if @project.last_repository_check_failed? = succeed '.' do %strong.cred failed See @@ -125,7 +125,7 @@ passed. .form-group - = f.submit 'Trigger repo check', class: 'btn btn-primary' + = f.submit 'Trigger repository check', class: 'btn btn-primary' .col-md-6 - if @group diff --git a/app/views/repo_check_mailer/notify.text.haml b/app/views/repo_check_mailer/notify.text.haml deleted file mode 100644 index bdf8c2ad675..00000000000 --- a/app/views/repo_check_mailer/notify.text.haml +++ /dev/null @@ -1,3 +0,0 @@ -#{@message}. -\ -View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)} diff --git a/app/views/repo_check_mailer/notify.html.haml b/app/views/repository_check_mailer/notify.html.haml index ef0016f9d3e..df16f503570 100644 --- a/app/views/repo_check_mailer/notify.html.haml +++ b/app/views/repository_check_mailer/notify.html.haml @@ -2,4 +2,4 @@ #{@message}. %p - = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1) + = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repository_check_failed: 1) diff --git a/app/views/repository_check_mailer/notify.text.haml b/app/views/repository_check_mailer/notify.text.haml new file mode 100644 index 00000000000..02f3f80288a --- /dev/null +++ b/app/views/repository_check_mailer/notify.text.haml @@ -0,0 +1,3 @@ +#{@message}. +\ +View details: #{admin_namespaces_projects_url(last_repository_check_failed: 1)} diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index fcccb9ea669..667fff031dd 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -4,9 +4,9 @@ class AdminEmailWorker sidekiq_options retry: false # this job auto-repeats via sidekiq-cron def perform - repo_check_failed_count = Project.where(last_repo_check_failed: true).count - return if repo_check_failed_count.zero? + repository_check_failed_count = Project.where(last_repository_check_failed: true).count + return if repository_check_failed_count.zero? - RepoCheckMailer.notify(repo_check_failed_count).deliver_now + RepositoryCheckMailer.notify(repository_check_failed_count).deliver_now end end diff --git a/app/workers/repo_check_worker.rb b/app/workers/repository_check_worker.rb index 9be795309e0..2d75c8bafde 100644 --- a/app/workers/repo_check_worker.rb +++ b/app/workers/repository_check_worker.rb @@ -1,4 +1,4 @@ -class RepoCheckWorker +class RepositoryCheckWorker include Sidekiq::Worker RUN_TIME = 3600 @@ -11,14 +11,14 @@ class RepoCheckWorker # This loop will break after a little more than one hour ('a little # more' because `git fsck` may take a few minutes), or if it runs out of # projects to check. By default sidekiq-cron will start a new - # RepoCheckWorker each hour so that as long as there are repositories to + # RepositoryCheckWorker each hour so that as long as there are repositories to # check, only one (or two) will be checked at a time. project_ids.each do |project_id| break if Time.now - start >= RUN_TIME next if !try_obtain_lease(project_id) - SingleRepoCheckWorker.new.perform(project_id) + SingleRepositoryCheckWorker.new.perform(project_id) end end @@ -29,16 +29,16 @@ class RepoCheckWorker # an array of IDs instead. def project_ids limit = 10_000 - never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit). + never_checked_projects = Project.where('last_repository_check_at IS NULL').limit(limit). pluck(:id) - old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago). - reorder('last_repo_check_at ASC').limit(limit).pluck(:id) + old_check_projects = Project.where('last_repository_check_at < ?', 1.week.ago). + reorder('last_repository_check_at ASC').limit(limit).pluck(:id) never_checked_projects + old_check_projects end def try_obtain_lease(id) lease = Gitlab::ExclusiveLease.new( - "project_repo_check:#{id}", + "project_repository_check:#{id}", timeout: RUN_TIME ) lease.try_obtain diff --git a/app/workers/single_repo_check_worker.rb b/app/workers/single_repository_check_worker.rb index f8b245247c5..d9eed9bd708 100644 --- a/app/workers/single_repo_check_worker.rb +++ b/app/workers/single_repository_check_worker.rb @@ -1,4 +1,4 @@ -class SingleRepoCheckWorker +class SingleRepositoryCheckWorker include Sidekiq::Worker sidekiq_options retry: false @@ -21,14 +21,14 @@ class SingleRepoCheckWorker output, status = Gitlab::Popen.popen(cmd) return true if status.zero? - Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}") + Gitlab::RepositoryCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}") false end def update(project, success:) project.update_columns( - last_repo_check_failed: !success, - last_repo_check_at: Time.now, + last_repository_check_failed: !success, + last_repository_check_at: Time.now, ) end end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index cf20fc9c63a..4fbef653bc1 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -157,7 +157,7 @@ production: &base cron: "0 0 * * *" # Periodically run 'git fsck' on all repositories. If started more than # once per hour you will have concurrent 'git fsck' jobs. - repo_check_worker: + repository_check_worker: cron: "20 * * * *" # Send admin emails once a day admin_email_worker: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 8240978ef06..23771553c45 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -239,9 +239,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' -Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *' -Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker' +Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *' +Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheckWorker' Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' diff --git a/config/routes.rb b/config/routes.rb index fad8600b77d..c0ed99b1964 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -264,11 +264,11 @@ Rails.application.routes.draw do member do put :transfer - post :repo_check + post :repository_check end collection do - put :clear_repo_check_states + put :clear_repository_check_states end resources :runner_projects diff --git a/db/migrate/20160315135439_project_add_repo_check.rb b/db/migrate/20160315135439_project_add_repo_check.rb deleted file mode 100644 index ba7ddc9ecb4..00000000000 --- a/db/migrate/20160315135439_project_add_repo_check.rb +++ /dev/null @@ -1,6 +0,0 @@ -class ProjectAddRepoCheck < ActiveRecord::Migration - def change - add_column :projects, :last_repo_check_failed, :boolean, default: false - add_column :projects, :last_repo_check_at, :datetime - end -end diff --git a/db/migrate/20160315135439_project_add_repository_check.rb b/db/migrate/20160315135439_project_add_repository_check.rb new file mode 100644 index 00000000000..5a0859a30b2 --- /dev/null +++ b/db/migrate/20160315135439_project_add_repository_check.rb @@ -0,0 +1,6 @@ +class ProjectAddRepositoryCheck < ActiveRecord::Migration + def change + add_column :projects, :last_repository_check_failed, :boolean, default: false + add_column :projects, :last_repository_check_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index d2c183f968b..53509956888 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -732,8 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do t.boolean "public_builds", default: true, null: false t.string "main_language" t.integer "pushes_since_gc", default: 0 - t.boolean "last_repo_check_failed", default: false - t.datetime "last_repo_check_at" + t.boolean "last_repository_check_failed", default: false + t.datetime "last_repository_check_at" end add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree diff --git a/doc/administration/repo_checks.md b/doc/administration/repository_checks.md index 81087f3ac1c..77f28209f2f 100644 --- a/doc/administration/repo_checks.md +++ b/doc/administration/repository_checks.md @@ -1,4 +1,4 @@ -# Repo checks +# Repository checks _**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_ @@ -13,10 +13,10 @@ output on the admin log page under 'repocheck.log'. ## Periodical checks -GitLab periodically runs a repo check on all project repositories and +GitLab periodically runs a repository check on all project repositories and wiki repositories in order to detect data corruption problems. A project will be checked no more than once per week. If any projects -fail their repo checks all GitLab administrators will receive an email +fail their repository checks all GitLab administrators will receive an email notification of the situation. This notification is sent out no more than once a day. @@ -27,27 +27,27 @@ schedule in gitlab.yml. ``` # For omnibus installations, in /etc/gitlab/gitlab.rb: -gitlab_rails['cron_jobs_repo_check_worker_cron'] = '' +gitlab_rails['cron_jobs_repository_check_worker_cron'] = '' ``` ``` # For installations from source, in config/gitlab.yml: cron_jobs: - repo_check_worker: + repository_check_worker: cron: "" ``` ## What to do if a check failed -If the repo check fails for some repository you should look up the error +If the repository check fails for some repository you should look up the error in repocheck.log (in the admin panel or on disk; see `/var/log/gitlab/gitlab-rails` for Omnibus installations or `/home/git/gitlab/log` for installations from source). Once you have -resolved the issue use the admin panel to trigger a new repo check on +resolved the issue use the admin panel to trigger a new repository check on the project. This will clear the 'check failed' state. -If for some reason the periodical repo check caused a lot of false -alarms you can choose to clear ALL repo check states from the admin +If for some reason the periodical repository check caused a lot of false +alarms you can choose to clear ALL repository check states from the admin project index page. --- diff --git a/lib/gitlab/repo_check_logger.rb b/lib/gitlab/repository_check_logger.rb index 9409f68722c..485b596ca57 100644 --- a/lib/gitlab/repo_check_logger.rb +++ b/lib/gitlab/repository_check_logger.rb @@ -1,5 +1,5 @@ module Gitlab - class RepoCheckLogger < Gitlab::Logger + class RepositoryCheckLogger < Gitlab::Logger def self.file_name_noext 'repocheck' end diff --git a/spec/features/admin/admin_projects_spec.rb b/spec/features/admin/admin_projects_spec.rb index e3991d48ed6..95a230a72c3 100644 --- a/spec/features/admin/admin_projects_spec.rb +++ b/spec/features/admin/admin_projects_spec.rb @@ -33,33 +33,33 @@ describe "Admin Projects", feature: true do end end - feature 'repo checks' do - scenario 'trigger repo check' do + feature 'repository checks' do + scenario 'trigger repository check' do visit_admin_project_page - page.within('.repo-check') do - click_button 'Trigger repo check' + page.within('.repository-check') do + click_button 'Trigger repository check' end - expect(page).to have_content('Repo check was triggered') + expect(page).to have_content('Repository check was triggered') end - scenario 'see failed repo check' do - @project.update_column(:last_repo_check_failed, true) + scenario 'see failed repository check' do + @project.update_column(:last_repository_check_failed, true) visit_admin_project_page - expect(page).to have_content('Last repo check failed') + expect(page).to have_content('Last repository check failed') end - scenario 'clear repo checks', js: true do - @project.update_column(:last_repo_check_failed, true) + scenario 'clear repository checks', js: true do + @project.update_column(:last_repository_check_failed, true) visit admin_namespaces_projects_path - page.within('.repo-check-states') do + page.within('.repository-check-states') do click_link 'Clear all' # pop-up should be auto confirmed end - expect(@project.reload.last_repo_check_failed).to eq(false) + expect(@project.reload.last_repository_check_failed).to eq(false) end end diff --git a/spec/mailers/repo_check_mailer_spec.rb b/spec/mailers/repository_check_mailer_spec.rb index d49a6ae0c05..6ae9a93aaac 100644 --- a/spec/mailers/repo_check_mailer_spec.rb +++ b/spec/mailers/repository_check_mailer_spec.rb @@ -1,6 +1,6 @@ require 'rails_helper' -describe RepoCheckMailer do +describe RepositoryCheckMailer do include EmailSpec::Matchers describe '.notify' do diff --git a/spec/workers/repo_check_worker_spec.rb b/spec/workers/repo_check_worker_spec.rb deleted file mode 100644 index 7ef3eba9ac5..00000000000 --- a/spec/workers/repo_check_worker_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -require 'spec_helper' - -describe RepoCheckWorker do - subject { RepoCheckWorker.new } - - it 'prefers projects that have never been checked' do - projects = 3.times.map { create(:project) } - projects[0].update_column(:last_repo_check_at, 1.month.ago) - projects[2].update_column(:last_repo_check_at, 3.weeks.ago) - - expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id)) - end - - it 'sorts projects by last_repo_check_at' do - projects = 3.times.map { create(:project) } - projects[0].update_column(:last_repo_check_at, 2.weeks.ago) - projects[1].update_column(:last_repo_check_at, 1.month.ago) - projects[2].update_column(:last_repo_check_at, 3.weeks.ago) - - expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id)) - end - - it 'excludes projects that were checked recently' do - projects = 3.times.map { create(:project) } - projects[0].update_column(:last_repo_check_at, 2.days.ago) - projects[1].update_column(:last_repo_check_at, 1.month.ago) - projects[2].update_column(:last_repo_check_at, 3.days.ago) - - expect(subject.perform).to eq([projects[1].id]) - end -end diff --git a/spec/workers/repository_check_worker_spec.rb b/spec/workers/repository_check_worker_spec.rb new file mode 100644 index 00000000000..d1849321f56 --- /dev/null +++ b/spec/workers/repository_check_worker_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe RepositoryCheckWorker do + subject { RepositoryCheckWorker.new } + + it 'prefers projects that have never been checked' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repository_check_at, 1.month.ago) + projects[2].update_column(:last_repository_check_at, 3.weeks.ago) + + expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id)) + end + + it 'sorts projects by last_repository_check_at' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repository_check_at, 2.weeks.ago) + projects[1].update_column(:last_repository_check_at, 1.month.ago) + projects[2].update_column(:last_repository_check_at, 3.weeks.ago) + + expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id)) + end + + it 'excludes projects that were checked recently' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repository_check_at, 2.days.ago) + projects[1].update_column(:last_repository_check_at, 1.month.ago) + projects[2].update_column(:last_repository_check_at, 3.days.ago) + + expect(subject.perform).to eq([projects[1].id]) + end +end |