diff options
Diffstat (limited to 'app')
29 files changed, 275 insertions, 37 deletions
diff --git a/app/assets/javascripts/subscription.js.coffee b/app/assets/javascripts/subscription.js.coffee index e4b7a3172ec..1a430f3aa47 100644 --- a/app/assets/javascripts/subscription.js.coffee +++ b/app/assets/javascripts/subscription.js.coffee @@ -2,7 +2,7 @@ class @Subscription constructor: (container) -> $container = $(container) @url = $container.attr('data-url') - @subscribe_button = $container.find('.subscribe-button') + @subscribe_button = $container.find('.js-subscribe-button') @subscription_status = $container.find('.subscription-status') @subscribe_button.unbind('click').click(@toggleSubscription) diff --git a/app/assets/stylesheets/pages/issuable.scss b/app/assets/stylesheets/pages/issuable.scss index d9218e15095..6bd90a23620 100644 --- a/app/assets/stylesheets/pages/issuable.scss +++ b/app/assets/stylesheets/pages/issuable.scss @@ -173,12 +173,6 @@ } } - .subscribe-button { - span { - margin-top: 0; - } - } - &.right-sidebar-collapsed { /* Extra small devices (phones, less than 768px) */ display: none; diff --git a/app/assets/stylesheets/pages/labels.scss b/app/assets/stylesheets/pages/labels.scss index da20fa28802..e179bdf0048 100644 --- a/app/assets/stylesheets/pages/labels.scss +++ b/app/assets/stylesheets/pages/labels.scss @@ -117,7 +117,7 @@ padding: 6px; color: $gl-text-color; - &.subscribe-button { + &.label-subscribe-button { padding-left: 0; } } diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index f010436bd36..b4a28b8dd3f 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -19,6 +19,15 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to admin_runners_path end + def clear_repository_check_states + RepositoryCheck::ClearWorker.perform_async + + redirect_to( + admin_application_settings_path, + notice: 'Started asynchronous removal of all repository check states.' + ) + end + private def set_application_setting @@ -82,6 +91,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :akismet_enabled, :akismet_api_key, :email_author_in_body, + :repository_checks_enabled, restricted_visibility_levels: [], import_sources: [] ) diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb index c6b3105544a..87986fdf8b1 100644 --- a/app/controllers/admin/projects_controller.rb +++ b/app/controllers/admin/projects_controller.rb @@ -1,5 +1,5 @@ class Admin::ProjectsController < Admin::ApplicationController - before_action :project, only: [:show, :transfer] + before_action :project, only: [:show, :transfer, :repository_check] before_action :group, only: [:show, :transfer] def index @@ -8,6 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController @projects = @projects.where("projects.visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present? @projects = @projects.with_push if params[:with_push].present? @projects = @projects.abandoned if params[:abandoned].present? + @projects = @projects.where(last_repository_check_failed: true) if params[:last_repository_check_failed].present? @projects = @projects.non_archived unless params[:with_archived].present? @projects = @projects.search(params[:name]) if params[:name].present? @projects = @projects.sort(@sort = params[:sort]) @@ -30,6 +31,15 @@ class Admin::ProjectsController < Admin::ApplicationController redirect_to admin_namespace_project_path(@project.namespace, @project) end + def repository_check + RepositoryCheck::SingleRepositoryWorker.perform_async(@project.id) + + redirect_to( + admin_namespace_project_path(@project.namespace, @project), + notice: 'Repository check was triggered.' + ) + end + protected def project diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 97d53acde94..ce5c84ee9bc 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ require 'fogbugz' class ApplicationController < ActionController::Base include Gitlab::CurrentSettings + include Gitlab::GonHelper include GitlabRoutingHelper include PageLayoutHelper @@ -158,20 +159,6 @@ class ApplicationController < ActionController::Base end end - def add_gon_variables - gon.api_version = API::API.version - gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param - gon.max_file_size = current_application_settings.max_attachment_size - gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - - if current_user - gon.current_user_id = current_user.id - gon.api_token = current_user.private_token - end - end - def validate_user_service_ticket! return unless signed_in? && session[:service_tickets] diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index c1adc999567..ee4fcc4e360 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -40,6 +40,7 @@ class GroupsController < Groups::ApplicationController @last_push = current_user.recent_push if current_user @projects = @projects.includes(:namespace) + @projects = @projects.sorted_by_activity @projects = filter_projects(@projects) @projects = @projects.sort(@sort = params[:sort]) @projects = @projects.page(params[:page]) if params[:filter_projects].blank? diff --git a/app/controllers/oauth/applications_controller.rb b/app/controllers/oauth/applications_controller.rb index d1e4ac10f6c..c6bdd0602c1 100644 --- a/app/controllers/oauth/applications_controller.rb +++ b/app/controllers/oauth/applications_controller.rb @@ -1,9 +1,11 @@ class Oauth::ApplicationsController < Doorkeeper::ApplicationsController include Gitlab::CurrentSettings + include Gitlab::GonHelper include PageLayoutHelper before_action :verify_user_oauth_applications_enabled before_action :authenticate_user! + before_action :add_gon_variables layout 'profile' diff --git a/app/mailers/repository_check_mailer.rb b/app/mailers/repository_check_mailer.rb new file mode 100644 index 00000000000..2bff5b63cc4 --- /dev/null +++ b/app/mailers/repository_check_mailer.rb @@ -0,0 +1,14 @@ +class RepositoryCheckMailer < BaseMailer + def notify(failed_count) + if failed_count == 1 + @message = "One project failed its last repository check" + else + @message = "#{failed_count} projects failed their last repository check" + end + + mail( + to: User.admins.pluck(:email), + subject: @message + ) + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 052cd874733..36f88154232 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -153,7 +153,8 @@ class ApplicationSetting < ActiveRecord::Base require_two_factor_authentication: false, two_factor_grace_period: 48, recaptcha_enabled: false, - akismet_enabled: false + akismet_enabled: false, + repository_checks_enabled: true, ) end diff --git a/app/models/issue.rb b/app/models/issue.rb index e064b0f8b95..3f188e04770 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -106,7 +106,7 @@ class Issue < ActiveRecord::Base def related_branches project.repository.branch_names.select do |branch| - branch.end_with?("-#{iid}") + branch =~ /\A#{iid}-(?!\d+-stable)/i end end @@ -151,7 +151,7 @@ class Issue < ActiveRecord::Base end def to_branch_name - "#{title.parameterize}-#{iid}" + "#{iid}-#{title.parameterize}" end def can_be_worked_on?(current_user) diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb new file mode 100644 index 00000000000..c78c7f4aa0e --- /dev/null +++ b/app/models/oauth_access_token.rb @@ -0,0 +1,19 @@ +# == Schema Information +# +# Table name: oauth_access_tokens +# +# id :integer not null, primary key +# resource_owner_id :integer +# application_id :integer +# token :string not null +# refresh_token :string +# expires_in :integer +# revoked_at :datetime +# created_at :datetime not null +# scopes :string +# + +class OauthAccessToken < ActiveRecord::Base + belongs_to :resource_owner, class_name: 'User' + belongs_to :application, class_name: 'Doorkeeper::Application' +end diff --git a/app/services/merge_requests/build_service.rb b/app/services/merge_requests/build_service.rb index 6e9152e444e..fa34753c4fd 100644 --- a/app/services/merge_requests/build_service.rb +++ b/app/services/merge_requests/build_service.rb @@ -51,7 +51,7 @@ module MergeRequests # be interpreted as the use wants to close that issue on this project # Pattern example: 112-fix-mep-mep # Will lead to appending `Closes #112` to the description - if match = merge_request.source_branch.match(/-(\d+)\z/) + if match = merge_request.source_branch.match(/\A(\d+)-/) iid = match[1] closes_issue = "Closes ##{iid}" diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 2e734654466..79a27f4af7e 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -34,8 +34,9 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - # Apply new namespace id + # Apply new namespace id and visibility level project.namespace = new_namespace + project.visibility_level = new_namespace.visibility_level unless project.visibility_level_allowed_by_group? project.save! # Notifications @@ -56,7 +57,7 @@ module Projects Gitlab::UploadsTransfer.new.move_project(project.path, old_namespace.path, new_namespace.path) project.old_path_with_namespace = old_path - + SystemHooksService.new.execute_hooks_for(project, :transfer) true end diff --git a/app/services/system_note_service.rb b/app/services/system_note_service.rb index 658b086496f..82a0e2fd1f5 100644 --- a/app/services/system_note_service.rb +++ b/app/services/system_note_service.rb @@ -222,7 +222,7 @@ class SystemNoteService # Called when a branch is created from the 'new branch' button on a issue # Example note text: # - # "Started branch `issue-branch-button-201`" + # "Started branch `201-issue-branch-button`" def self.new_issue_branch(issue, project, author, branch) h = Gitlab::Routing.url_helpers link = h.namespace_project_compare_url(project.namespace, project, from: project.default_branch, to: branch) diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index a8cca1a81cb..555aea554f0 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -271,5 +271,24 @@ .col-sm-10 = f.text_field :sentry_dsn, class: 'form-control' + %fieldset + %legend Repository Checks + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :repository_checks_enabled do + = f.check_box :repository_checks_enabled + Enable Repository Checks + .help-block + GitLab will periodically run + %a{ href: 'https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html', target: 'blank' } 'git fsck' + in all project and wiki repositories to look for silent disk corruption issues. + .form-group + .col-sm-offset-2.col-sm-10 + = link_to 'Clear all repository checks', clear_repository_check_states_admin_application_settings_path, data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove" + .help-block + If you got a lot of false alarms from repository checks you can choose to clear all repository check information from the database. + + .form-actions = f.submit 'Save', class: 'btn btn-save' diff --git a/app/views/admin/logs/show.html.haml b/app/views/admin/logs/show.html.haml index af9fdeb0734..4b475a4d8fa 100644 --- a/app/views/admin/logs/show.html.haml +++ b/app/views/admin/logs/show.html.haml @@ -1,6 +1,7 @@ - page_title "Logs" - loggers = [Gitlab::GitLogger, Gitlab::AppLogger, - Gitlab::ProductionLogger, Gitlab::SidekiqLogger] + Gitlab::ProductionLogger, Gitlab::SidekiqLogger, + Gitlab::RepositoryCheckLogger] %ul.nav-links.log-tabs - loggers.each do |klass| %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml index d39c0f44031..aa07afa0d62 100644 --- a/app/views/admin/projects/index.html.haml +++ b/app/views/admin/projects/index.html.haml @@ -3,7 +3,7 @@ .row.prepend-top-default %aside.col-md-3 - .admin-filter + .panel.admin-filter = form_tag admin_namespaces_projects_path, method: :get, class: '' do .form-group = label_tag :name, 'Name:' @@ -38,7 +38,13 @@ %span.descr = visibility_level_icon(level) = label - %hr + %fieldset + %strong Problems + .checkbox + = label_tag :last_repository_check_failed do + = check_box_tag :last_repository_check_failed, 1, params[:last_repository_check_failed] + %span Last repository check failed + = hidden_field_tag :sort, params[:sort] = button_tag "Search", class: "btn submit btn-primary" = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel" diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index c638c32a654..73986d21bcf 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -5,6 +5,16 @@ %i.fa.fa-pencil-square-o Edit %hr +- if @project.last_repository_check_failed? + .row + .col-md-12 + .panel + .panel-heading.alert.alert-danger + Last repository check + = "(#{time_ago_in_words(@project.last_repository_check_at)} ago)" + failed. See + = link_to 'repocheck.log', admin_logs_path + for error messages. .row .col-md-6 .panel.panel-default @@ -95,6 +105,32 @@ .col-sm-offset-2.col-sm-10 = f.submit 'Transfer', class: 'btn btn-primary' + .panel.panel-default.repository-check + .panel-heading + Repository check + .panel-body + = form_for @project, url: repository_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f| + .form-group + - if @project.last_repository_check_at.nil? + This repository has never been checked. + - else + This repository was last checked + = @project.last_repository_check_at.to_s(:medium) + '.' + The check + - if @project.last_repository_check_failed? + = succeed '.' do + %strong.cred failed + See + = link_to 'repocheck.log', admin_logs_path + for error messages. + - else + passed. + + = link_to icon('question-circle'), help_page_path('administration', 'repository_checks') + + .form-group + = f.submit 'Trigger repository check', class: 'btn btn-primary' + .col-md-6 - if @group .panel.panel-default diff --git a/app/views/doorkeeper/applications/index.html.haml b/app/views/doorkeeper/applications/index.html.haml index 55f4a6f287d..0aff79749ef 100644 --- a/app/views/doorkeeper/applications/index.html.haml +++ b/app/views/doorkeeper/applications/index.html.haml @@ -68,7 +68,7 @@ %td= app.name %td= token.created_at %td= token.scopes - %td= render 'delete_form', application: app + %td= render 'doorkeeper/authorized_applications/delete_form', application: app - @authorized_anonymous_tokens.each do |token| %tr %td diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml index 6d872cd0b21..76a4f41193c 100644 --- a/app/views/projects/edit.html.haml +++ b/app/views/projects/edit.html.haml @@ -210,6 +210,7 @@ %li Be careful. Changing the project's namespace can have unintended side effects. %li You can only transfer the project to namespaces you manage. %li You will need to update your local repositories to point to the new location. + %li Project visibility level will be changed to match namespace rules when transfering to a group. .form-actions = f.submit 'Transfer project', class: "btn btn-remove js-confirm-danger", data: { "confirm-danger-message" => transfer_project_message(@project) } - else diff --git a/app/views/projects/labels/_label.html.haml b/app/views/projects/labels/_label.html.haml index 097a65969a6..8bf544b8371 100644 --- a/app/views/projects/labels/_label.html.haml +++ b/app/views/projects/labels/_label.html.haml @@ -14,7 +14,7 @@ .label-subscription{data: {url: toggle_subscription_namespace_project_label_path(@project.namespace, @project, label)}} .subscription-status{data: {status: label_subscription_status(label)}} - %a.subscribe-button.btn.action-buttons{data: {toggle: "tooltip"}} + %button.js-subscribe-button.label-subscribe-button.btn.action-buttons{ type: "button", data: { toggle: "tooltip" } } %span= label_subscription_toggle_button_text(label) - if can? current_user, :admin_label, @project diff --git a/app/views/repository_check_mailer/notify.html.haml b/app/views/repository_check_mailer/notify.html.haml new file mode 100644 index 00000000000..df16f503570 --- /dev/null +++ b/app/views/repository_check_mailer/notify.html.haml @@ -0,0 +1,5 @@ +%p + #{@message}. + +%p + = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repository_check_failed: 1) diff --git a/app/views/repository_check_mailer/notify.text.haml b/app/views/repository_check_mailer/notify.text.haml new file mode 100644 index 00000000000..02f3f80288a --- /dev/null +++ b/app/views/repository_check_mailer/notify.text.haml @@ -0,0 +1,3 @@ +#{@message}. +\ +View details: #{admin_namespaces_projects_url(last_repository_check_failed: 1)} diff --git a/app/views/shared/issuable/_sidebar.html.haml b/app/views/shared/issuable/_sidebar.html.haml index 94affa4b59a..56c8eaa0597 100644 --- a/app/views/shared/issuable/_sidebar.html.haml +++ b/app/views/shared/issuable/_sidebar.html.haml @@ -128,7 +128,7 @@ .title.hide-collapsed Notifications - subscribtion_status = subscribed ? 'subscribed' : 'unsubscribed' - %button.btn.btn-block.btn-gray.subscribe-button.hide-collapsed{:type => 'button'} + %button.btn.btn-block.btn-gray.js-subscribe-button.issuable-subscribe-button.hide-collapsed{ type: "button" } %span= subscribed ? 'Unsubscribe' : 'Subscribe' .subscription-status.hide-collapsed{data: {status: subscribtion_status}} .unsubscribed{class: ( 'hidden' if subscribed )} @@ -152,4 +152,4 @@ new LabelsSelect(); new IssuableContext('#{current_user.to_json(only: [:username, :id, :name])}'); new Subscription('.subscription') - new Sidebar();
\ No newline at end of file + new Sidebar(); diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb new file mode 100644 index 00000000000..667fff031dd --- /dev/null +++ b/app/workers/admin_email_worker.rb @@ -0,0 +1,12 @@ +class AdminEmailWorker + include Sidekiq::Worker + + sidekiq_options retry: false # this job auto-repeats via sidekiq-cron + + def perform + repository_check_failed_count = Project.where(last_repository_check_failed: true).count + return if repository_check_failed_count.zero? + + RepositoryCheckMailer.notify(repository_check_failed_count).deliver_now + end +end diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb new file mode 100644 index 00000000000..44b3145d50f --- /dev/null +++ b/app/workers/repository_check/batch_worker.rb @@ -0,0 +1,63 @@ +module RepositoryCheck + class BatchWorker + include Sidekiq::Worker + + RUN_TIME = 3600 + + sidekiq_options retry: false + + def perform + start = Time.now + + # This loop will break after a little more than one hour ('a little + # more' because `git fsck` may take a few minutes), or if it runs out of + # projects to check. By default sidekiq-cron will start a new + # RepositoryCheckWorker each hour so that as long as there are repositories to + # check, only one (or two) will be checked at a time. + project_ids.each do |project_id| + break if Time.now - start >= RUN_TIME + break unless current_settings.repository_checks_enabled + + next unless try_obtain_lease(project_id) + + SingleRepositoryWorker.new.perform(project_id) + end + end + + private + + # Project.find_each does not support WHERE clauses and + # Project.find_in_batches does not support ordering. So we just build an + # array of ID's. This is OK because we do it only once an hour, because + # getting ID's from Postgres is not terribly slow, and because no user + # has to sit and wait for this query to finish. + def project_ids + limit = 10_000 + never_checked_projects = Project.where('last_repository_check_at IS NULL').limit(limit). + pluck(:id) + old_check_projects = Project.where('last_repository_check_at < ?', 1.month.ago). + reorder('last_repository_check_at ASC').limit(limit).pluck(:id) + never_checked_projects + old_check_projects + end + + def try_obtain_lease(id) + # Use a 24-hour timeout because on servers/projects where 'git fsck' is + # super slow we definitely do not want to run it twice in parallel. + Gitlab::ExclusiveLease.new( + "project_repository_check:#{id}", + timeout: 24.hours + ).try_obtain + end + + def current_settings + # No caching of the settings! If we cache them and an admin disables + # this feature, an active RepositoryCheckWorker would keep going for up + # to 1 hour after the feature was disabled. + if Rails.env.test? + Gitlab::CurrentSettings.fake_application_settings + else + ApplicationSetting.current + end + end + end +end diff --git a/app/workers/repository_check/clear_worker.rb b/app/workers/repository_check/clear_worker.rb new file mode 100644 index 00000000000..b7202ddff34 --- /dev/null +++ b/app/workers/repository_check/clear_worker.rb @@ -0,0 +1,17 @@ +module RepositoryCheck + class ClearWorker + include Sidekiq::Worker + + sidekiq_options retry: false + + def perform + # Do small batched updates because these updates will be slow and locking + Project.select(:id).find_in_batches(batch_size: 100) do |batch| + Project.where(id: batch.map(&:id)).update_all( + last_repository_check_failed: nil, + last_repository_check_at: nil, + ) + end + end + end +end diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb new file mode 100644 index 00000000000..e54ae86d06c --- /dev/null +++ b/app/workers/repository_check/single_repository_worker.rb @@ -0,0 +1,36 @@ +module RepositoryCheck + class SingleRepositoryWorker + include Sidekiq::Worker + + sidekiq_options retry: false + + def perform(project_id) + project = Project.find(project_id) + project.update_columns( + last_repository_check_failed: !check(project), + last_repository_check_at: Time.now, + ) + end + + private + + def check(project) + # Use 'map do', not 'all? do', to prevent short-circuiting + [project.repository, project.wiki.repository].map do |repository| + git_fsck(repository.path_to_repo) + end.all? + end + + def git_fsck(path) + cmd = %W(nice git --git-dir=#{path} fsck) + output, status = Gitlab::Popen.popen(cmd) + + if status.zero? + true + else + Gitlab::RepositoryCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}") + false + end + end + end +end |