diff options
Diffstat (limited to 'lib')
49 files changed, 930 insertions, 269 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 254e1e786f5..5e93c129bc8 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -108,6 +108,7 @@ module API mount ::API::AccessRequests mount ::API::Applications mount ::API::AwardEmoji + mount ::API::Badges mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages @@ -139,6 +140,7 @@ module API mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/badges.rb b/lib/api/badges.rb new file mode 100644 index 00000000000..334948b2995 --- /dev/null +++ b/lib/api/badges.rb @@ -0,0 +1,134 @@ +module API + class Badges < Grape::API + include PaginationParams + + before { authenticate_non_get! } + + helpers ::API::Helpers::BadgesHelpers + + helpers do + def find_source_if_admin(source_type) + source = find_source(source_type, params[:id]) + + authorize_admin_source!(source_type, source) + + source + end + end + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The ID of a #{source_type}" + end + resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Gets a list of #{source_type} badges viewable by the authenticated user." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + use :pagination + end + get ":id/badges" do + source = find_source(source_type, params[:id]) + + present_badges(source, paginate(source.badges)) + end + + desc "Preview a badge from a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::BasicBadgeDetails + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + get ":id/badges/render" do + authenticate! + + source = find_source_if_admin(source_type) + + badge = ::Badges::BuildService.new(declared_params(include_missing: false)) + .execute(source) + + if badge.valid? + present_badges(source, badge, with: Entities::BasicBadgeDetails) + else + render_validation_error!(badge) + end + end + + desc "Gets a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + get ":id/badges/:badge_id" do + source = find_source(source_type, params[:id]) + badge = find_badge(source) + + present_badges(source, badge) + end + + desc "Adds a badge to a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + requires :link_url, type: String, desc: 'URL of the badge link' + requires :image_url, type: String, desc: 'URL of the badge image' + end + post ":id/badges" do + source = find_source_if_admin(source_type) + + badge = ::Badges::CreateService.new(declared_params(include_missing: false)).execute(source) + + if badge.persisted? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc "Updates a badge of a #{source_type}." do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::Badge + end + params do + optional :link_url, type: String, desc: 'URL of the badge link' + optional :image_url, type: String, desc: 'URL of the badge image' + end + put ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + + badge = ::Badges::UpdateService.new(declared_params(include_missing: false)) + .execute(find_badge(source)) + + if badge.valid? + present_badges(source, badge) + else + render_validation_error!(badge) + end + end + + desc 'Removes a badge from a project or group.' do + detail 'This feature was introduced in GitLab 10.6.' + end + params do + requires :badge_id, type: Integer, desc: 'The badge ID' + end + delete ":id/badges/:badge_id" do + source = find_source_if_admin(source_type) + badge = find_badge(source) + + if badge.is_a?(GroupBadge) && source.is_a?(Project) + error!('To delete a Group badge please use the Group endpoint', 403) + end + + destroy_conditionally!(badge) + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 1794207e29b..13cfba728fa 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -16,6 +16,10 @@ module API render_api_error!('The branch refname is invalid', 400) end end + + params :filter_params do + optional :search, type: String, desc: 'Return list of branches matching the search criteria' + end end params do @@ -27,15 +31,23 @@ module API end params do use :pagination + use :filter_params end get ':id/repository/branches' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') repository = user_project.repository - branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) + + branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names + present( + paginate(::Kaminari.paginate_array(branches)), + with: Entities::Branch, + project: user_project, + merged_branch_names: merged_branch_names + ) end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0c8ec7dd5f5..f39906270d8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -91,6 +91,21 @@ module API expose :created_at end + class ProjectExportStatus < ProjectIdentity + include ::API::Helpers::RelatedResourcesHelpers + + expose :export_status + expose :_links, if: lambda { |project, _options| project.export_status == :finished } do + expose :api_url do |project| + expose_url(api_v4_projects_export_download_path(id: project.id)) + end + + expose :web_url do |project| + Gitlab::Routing.url_helpers.download_export_project_url(project) + end + end + end + class ProjectImportStatus < ProjectIdentity expose :import_status @@ -1235,5 +1250,23 @@ module API expose :startline expose :project_id end + + class BasicBadgeDetails < Grape::Entity + expose :link_url + expose :image_url + expose :rendered_link_url do |badge, options| + badge.rendered_link_url(options.fetch(:project, nil)) + end + expose :rendered_image_url do |badge, options| + badge.rendered_image_url(options.fetch(:project, nil)) + end + end + + class Badge < BasicBadgeDetails + expose :id + expose :kind do |badge| + badge.type == 'ProjectBadge' ? 'project' : 'group' + end + end end end diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb new file mode 100644 index 00000000000..1f8afbf3c90 --- /dev/null +++ b/lib/api/helpers/badges_helpers.rb @@ -0,0 +1,28 @@ +module API + module Helpers + module BadgesHelpers + include ::API::Helpers::MembersHelpers + + def find_badge(source) + source.badges.find(params[:badge_id]) + end + + def present_badges(source, records, options = {}) + entity_type = options[:with] || Entities::Badge + badge_params = badge_source_params(source).merge(with: entity_type) + + present records, badge_params + end + + def badge_source_params(source) + project = if source.is_a?(Project) + source + else + GroupProjectsFinder.new(group: source, current_user: current_user).execute.first + end + + { project: project } + end + end + end +end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b6c278c89d0..f74b3b26802 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -32,6 +32,8 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 2a8fa7659bf..47e5eeab31d 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -2,20 +2,28 @@ module API class JobArtifacts < Grape::API before { authenticate_non_get! } + # EE::API::JobArtifacts would override the following helpers + helpers do + def authorize_download_artifacts! + authorize_read_builds! + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Download the artifacts file from a job' do + desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.10' end params do requires :ref_name, type: String, desc: 'The ref from repository' requires :job, type: String, desc: 'The name for the job' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do - authorize_read_builds! + authorize_download_artifacts! builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) @@ -23,14 +31,15 @@ module API present_artifacts!(latest_build.artifacts_file) end - desc 'Download the artifacts file from a job' do + desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.5' end params do requires :job_id, type: Integer, desc: 'The ID of a job' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! + authorize_download_artifacts! build = find_build!(params[:job_id]) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 16d0f005f21..8c02972b421 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -42,6 +42,8 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb new file mode 100644 index 00000000000..6ec2626df1a --- /dev/null +++ b/lib/api/project_export.rb @@ -0,0 +1,41 @@ +module API + class ProjectExport < Grape::API + before do + not_found! unless Gitlab::CurrentSettings.project_export_enabled? + authorize_admin_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get export status' do + detail 'This feature was introduced in GitLab 10.6.' + success Entities::ProjectExportStatus + end + get ':id/export' do + present user_project, with: Entities::ProjectExportStatus + end + + desc 'Download export' do + detail 'This feature was introduced in GitLab 10.6.' + end + get ':id/export/download' do + path = user_project.export_project_path + + render_api_error!('404 Not found or has expired', 404) unless path + + present_file!(path, File.basename(path), 'application/gzip') + end + + desc 'Start export' do + detail 'This feature was introduced in GitLab 10.6.' + end + post ':id/export' do + user_project.add_export_job(current_user: current_user) + + accepted! + end + end + end +end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 91cdc564002..7e6c33ec33d 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -204,6 +204,7 @@ module API optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file) optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) end @@ -224,7 +225,7 @@ module API expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, expire_in: expire_in) + job.build_job_artifacts_archive(project: job.project, file_type: :archive, file: artifacts, file_sha256: params['file.sha256'], expire_in: expire_in) job.build_job_artifacts_metadata(project: job.project, file_type: :metadata, file: metadata, expire_in: expire_in) if metadata job.artifacts_expire_in = expire_in diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index b8d2673c1a6..75b64ae9af2 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -25,8 +25,8 @@ module Banzai # period or comma for punctuation without those characters being included # in the generated link. # - # Rubular: http://rubular.com/r/cxjPyZc7Sb - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://\S+)(?<!,|\.)} + # Rubular: http://rubular.com/r/JzPhi6DCZp + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -35,53 +35,19 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') - and not(starts-with(., 'http')) - and not(starts-with(., 'ftp')) ]).freeze + PUNCTUATION_PAIRS = { + "'" => "'", + '"' => '"', + ')' => '(', + ']' => '[', + '}' => '{' + }.freeze + def call return doc if context[:autolink] == false - rinku_parse - text_parse - end - - private - - # Run the text through Rinku as a first pass - # - # This will quickly autolink http(s) and ftp links. - # - # `@doc` will be re-parsed with the HTML String from Rinku. - def rinku_parse - # Convert the options from a Hash to a String that Rinku expects - options = tag_options(link_options) - - # NOTE: We don't parse email links because it will erroneously match - # external Commit and CommitRange references. - # - # The final argument tells Rinku to link short URLs that don't include a - # period (e.g., http://localhost:3000/) - rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) - - return if rinku == html - - # Rinku returns a String, so parse it back to a Nokogiri::XML::Document - # for further processing. - @doc = parse_html(rinku) - end - - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - - # Autolinks any text matching LINK_PATTERN that Rinku didn't already - # replace - def text_parse doc.xpath(TEXT_QUERY).each do |node| content = node.to_html @@ -97,6 +63,16 @@ module Banzai doc end + private + + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + def autolink_match(match) # start by stripping out dangerous links begin @@ -112,12 +88,30 @@ module Banzai match.gsub!(/((?:&[\w#]+;)+)\z/, '') dropped = ($1 || '').html_safe + # To match the behaviour of Rinku, if the matched link ends with a + # closing part of a matched pair of punctuation, we remove that trailing + # character unless there are an equal number of closing and opening + # characters in the link. + if match.end_with?(*PUNCTUATION_PAIRS.keys) + close_character = match[-1] + close_count = match.count(close_character) + open_character = PUNCTUATION_PAIRS[close_character] + open_count = match.count(open_character) + + if open_count != close_count || open_character == close_character + dropped += close_character + match = match[0..-2] + end + end + options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + content_tag(:a, match.html_safe, options) + dropped end def autolink_filter(text) - text.gsub(LINK_PATTERN) { |match| autolink_match(match) } + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + autolink_match(link) + end end def link_options diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 86393ee254d..f5ccf952cf9 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -40,8 +40,8 @@ module Gitlab end def find_with_user_password(login, password) - # Avoid resource intensive login checks if password is not provided - return unless password.present? + # Avoid resource intensive checks if login credentials are not provided + return unless login.present? && password.present? # Nothing to do here if internal auth is disabled and LDAP is # not configured @@ -50,14 +50,26 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - Gitlab::Auth::LDAP::Authentication.login(login, password) - elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user if user.active? && user.valid_password?(password) + return if user && !user.active? + + authenticators = [] + + if user + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, 'database') + + # Add authenticators for all identities if user is not nil + user&.identities&.each do |identity| + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, identity.provider) + end + else + # If no user is provided, try LDAP. + # LDAP users are only authenticated via LDAP + authenticators << Gitlab::Auth::LDAP::Authentication end + + authenticators.compact! + + user if authenticators.find { |auth| auth.login(login, password) } end end diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb new file mode 100644 index 00000000000..260a77058a4 --- /dev/null +++ b/lib/gitlab/auth/database/authentication.rb @@ -0,0 +1,16 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module Database + class Authentication < Gitlab::Auth::OAuth::Authentication + def login(login, password) + return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + + user&.valid_password?(password) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index cbb9cf4bb9c..e70c3ab6b46 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -7,7 +7,7 @@ module Gitlab module Auth module LDAP - class Authentication + class Authentication < Gitlab::Auth::OAuth::Authentication def self.login(login, password) return unless Gitlab::Auth::LDAP::Config.enabled? return unless login.present? && password.present? @@ -28,11 +28,7 @@ module Gitlab Gitlab::Auth::LDAP::Config.providers end - attr_accessor :provider, :ldap_user - - def initialize(provider) - @provider = provider - end + attr_accessor :ldap_user def login(login, password) @ldap_user = adapter.bind_as( @@ -62,7 +58,7 @@ module Gitlab end def user - return nil unless ldap_user + return unless ldap_user Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb new file mode 100644 index 00000000000..ed03b9f8b40 --- /dev/null +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -0,0 +1,21 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module OAuth + class Authentication + attr_reader :provider, :user + + def initialize(provider, user = nil) + @provider = provider + @user = user + end + + def login(login, password) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index f8ab8ee1388..5fb61ffe00d 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -8,11 +8,28 @@ module Gitlab "google_oauth2" => "Google" }.freeze + def self.authentication(user, provider) + return unless user + return unless enabled?(provider) + + authenticator = + case provider + when /^ldap/ + Gitlab::Auth::LDAP::Authentication + when 'database' + Gitlab::Auth::Database::Authentication + end + + authenticator&.new(provider, user) + end + def self.providers Devise.omniauth_providers end def self.enabled?(name) + return true if name == 'database' + providers.include?(name.to_sym) end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index acd785bb02d..b6a96081278 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -161,7 +161,7 @@ module Gitlab def find_by_uid_and_provider identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take - identity && identity.user + identity&.user end def build_new_user diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d19a2519803..d5e17a123df 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,27 +17,11 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") - ensure - if pipeline.builds.where(stage_id: nil).any? - invalid_builds_counter.increment(node: hostname) - end end def break? !pipeline.persisted? end - - private - - def invalid_builds_counter - @counter ||= Gitlab::Metrics - .counter(:gitlab_ci_invalid_builds_total, - 'Invalid builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f2e5124c8a8..c5356af2f02 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,6 +1,8 @@ module Gitlab module Ci class Trace + ArchiveError = Class.new(StandardError) + attr_reader :job delegate :old_trace, to: :job @@ -93,8 +95,52 @@ module Gitlab job.erase_old_trace! end + def archive! + raise ArchiveError, 'Already archived' if trace_artifact + raise ArchiveError, 'Job is not finished yet' unless job.complete? + + if current_path + File.open(current_path) do |stream| + archive_stream!(stream) + FileUtils.rm(current_path) + end + elsif old_trace + StringIO.new(old_trace, 'rb').tap do |stream| + archive_stream!(stream) + job.erase_old_trace! + end + end + end + private + def archive_stream!(stream) + clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| + create_job_trace!(job, clone_path) + end + end + + def clone_file!(src_stream, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.touch(temp_path) + size = IO.copy_stream(src_stream, temp_path) + raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size + + yield(temp_path) + end + end + + def create_job_trace!(job, path) + File.open(path) do |stream| + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream) + end + end + def ensure_path return current_path if current_path diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9576d5a3fd8..02d3763514e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -23,7 +23,7 @@ module Gitlab mr_events = event_counts(date_from, :merge_requests) .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") note_events = event_counts(date_from, :merge_requests) - .having(action: [Event::COMMENTED], target_type: "Note") + .having(action: [Event::COMMENTED], target_type: %w(Note DiffNote)) union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b2fca2c16de..eabcf46cf58 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -238,9 +238,9 @@ module Gitlab self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end - @loaded_all_data = false # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data + @loaded_all_data = @loaded_size == size end def binary? @@ -255,10 +255,15 @@ module Gitlab # memory as a Ruby string. def load_all_data!(repository) return if @data == '' # don't mess with submodule blobs - return @data if @loaded_all_data - Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - @data = begin + # Even if we return early, recalculate wether this blob is binary in + # case a blob was initialized as text but the full data isn't + @binary = nil + + return if @loaded_all_data + + @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + begin if is_enabled repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else @@ -269,7 +274,6 @@ module Gitlab @loaded_all_data = true @loaded_size = @data.bytesize - @binary = nil end def name diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index ae7e88f0503..6351cfb83e3 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Branch < Ref + STALE_BRANCH_THRESHOLD = 3.months + def self.find(repo, branch_name) if branch_name.is_a?(Gitlab::Git::Branch) branch_name @@ -12,6 +14,18 @@ module Gitlab def initialize(repository, name, target, target_commit) super(repository, name, target, target_commit) end + + def active? + self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago + end + + def stale? + !active? + end + + def state + active? ? :active : :stale + end end end end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 48434047fce..b9e5cf258f4 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,6 +7,28 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) + @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + else + git_new_pointers(object_limit, not_in) + end + end + end + + def all_pointers + @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) + else + git_all_pointers + end + end + end + + private + + def git_new_pointers(object_limit, not_in) @new_pointers ||= begin rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| object_ids = object_ids.take(object_limit) if object_limit @@ -16,14 +38,12 @@ module Gitlab end end - def all_pointers + def git_all_pointers rev_list.all_objects(require_path: true) do |object_ids| Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) end end - private - def rev_list Gitlab::Git::RevList.new(@repository, newrev: @newrev) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7c373ccd6f..21c79a7a550 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -479,9 +479,8 @@ module Gitlab raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") end - # TODO support options[:all] in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/1049 gitaly_migrate(:find_commits) do |is_enabled| - if is_enabled && !options[:all] + if is_enabled gitaly_commit_client.find_commits(options) else raw_log(options).map { |c| Commit.decorate(self, c) } @@ -508,9 +507,8 @@ module Gitlab def count_commits(options) count_commits_options = process_count_commits_options(options) - # TODO add support for options[:all] in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/1050 gitaly_migrate(:count_commits) do |is_enabled| - if is_enabled && !options[:all] + if is_enabled count_commits_by_gitaly(count_commits_options) else count_commits_by_shelling_out(count_commits_options) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index dfa0fa43b0f..28554208984 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -45,16 +45,7 @@ module Gitlab response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) - response.flat_map do |message| - message.lfs_pointers.map do |lfs_pointer| - Gitlab::Git::Blob.new( - id: lfs_pointer.oid, - size: lfs_pointer.size, - data: lfs_pointer.data, - binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) - ) - end - end + map_lfs_pointers(response) end def get_blobs(revision_paths, limit = -1) @@ -80,6 +71,50 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end + + def get_new_lfs_pointers(revision, limit, not_in) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + limit: limit || 0 + ) + + if not_in.nil? || not_in == :all + request.not_in_all = true + else + request.not_in_refs += not_in + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + + map_lfs_pointers(response) + end + + def get_all_lfs_pointers(revision) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + + map_lfs_pointers(response) + end + + private + + def map_lfs_pointers(response) + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ad0bf1d060..456a8a1a2d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -134,7 +134,8 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: encode_binary(ref) + revision: encode_binary(ref), + all: !!options[:all] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -269,6 +270,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], + all: !!options[:all], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92f0e0402a8..a7e055ac444 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,6 +20,7 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.test_env = Rails.env.test? + gon.suggested_label_colors = LabelsHelper.suggested_colors if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index bdc0f04b56b..3772ef11c7f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -18,7 +18,10 @@ module Gitlab 'uk' => 'Українська', 'ja' => '日本語', 'ko' => '한국어', - 'nl_NL' => 'Nederlands' + 'nl_NL' => 'Nederlands', + 'tr_TR' => 'Türkçe', + 'id_ID' => 'Bahasa Indonesia', + 'fil_PH' => 'Filipino' }.freeze def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index af203ff711d..b713fa7e1cd 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.2'.freeze + VERSION = '0.2.3'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9f404003125..4bdd01f5e94 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -65,6 +65,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :project_badges # Only include the following attributes for the models specified. included_attributes: @@ -125,6 +126,8 @@ excluded_attributes: - :when push_event_payload: - :event_id + project_badges: + - :group_id methods: labels: @@ -147,3 +150,5 @@ methods: - :action push_event_payload: - :action + project_badges: + - :type diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a00795f553e..c38df9102eb 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab @archive_file = project.import_source @current_user = project.creator @project = project - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + @shared = project.import_export_shared end def execute diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 759833a5ee5..cf6b7e306dd 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -16,7 +16,8 @@ module Gitlab priorities: :label_priorities, auto_devops: :project_auto_devops, label: :project_label, - custom_attributes: 'ProjectCustomAttribute' }.freeze + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index b34cafc6876..3d3d998a6a3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,13 +1,17 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts + attr_reader :errors, :project - def initialize(opts) - @opts = opts + def initialize(project) + @project = project @errors = [] end + def active_export_count + Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) } + end + def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -31,11 +35,11 @@ module Gitlab private def relative_path - File.join(opts[:relative_path], SecureRandom.hex) + File.join(relative_archive_path, SecureRandom.hex) end def relative_archive_path - File.join(opts[:relative_path], '..') + @project.disk_path end def error_out(message, caller) diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..d9d5f90596f 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,90 +1,19 @@ module Gitlab module Middleware class ReadOnly - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze API_VERSIONS = (3..4) + def self.internal_routes + @internal_routes ||= + API_VERSIONS.map { |version| "api/v#{version}/internal" } + end + def initialize(app) @app = app - @whitelisted = internal_routes end def call(env) - @env = env - @route_hash = nil - - if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') - error_message = 'You cannot do writing operations on a read-only GitLab instance' - - if json_request? - return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] - else - rack_flash.alert = error_message - rack_session['flash'] = rack_flash.to_session_value - - return [301, { 'Location' => last_visited_url }, []] - end - end - - @app.call(env) - end - - private - - def internal_routes - API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } - end - - def disallowed_request? - DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes - end - - def json_request? - request.media_type == APPLICATION_JSON - end - - def rack_flash - @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) - end - - def rack_session - @env['rack.session'] - end - - def request - @env['rack.request'] ||= Rack::Request.new(@env) - end - - def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url - end - - def route_hash - @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} - end - - def whitelisted_routes - grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - - def grack_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') - - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' - end - - def lfs_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') - - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb new file mode 100644 index 00000000000..45b644e6510 --- /dev/null +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -0,0 +1,86 @@ +module Gitlab + module Middleware + class ReadOnly + class Controller + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + + def initialize(app, env) + @app = app + @env = env + end + + def call + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + + if json_request? + return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] + else + rack_flash.alert = ERROR_MESSAGE + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(@env) + end + + private + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && + !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end + end +end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb new file mode 100644 index 00000000000..f8d0a135965 --- /dev/null +++ b/lib/gitlab/middleware/release_env.rb @@ -0,0 +1,14 @@ +module Gitlab + module Middleware + # Some of middleware would hold env for no good reason even after the + # request had already been processed, and we could not garbage collect + # them due to this. Put this middleware as the first middleware so that + # it would clear the env after the request is done, allowing GC gets a + # chance to release memory for the last request. + ReleaseEnv = Struct.new(:app) do + def call(env) + app.call(env).tap { env.clear } + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index cf0935dbd9a..29277ec6481 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,8 +29,18 @@ module Gitlab @blobs_count ||= blobs.count end - def notes_count - @notes_count ||= notes.count + def limited_notes_count + return @limited_notes_count if defined?(@limited_notes_count) + + types = %w(issue merge_request commit snippet) + @limited_notes_count = 0 + + types.each do |type| + @limited_notes_count += notes_finder(type).limit(count_limit).count + break if @limited_notes_count >= count_limit + end + + @limited_notes_count end def wiki_blobs_count @@ -72,11 +82,12 @@ module Gitlab end def single_commit_result? - commits_count == 1 && total_result_count == 1 - end + return false if commits_count != 1 - def total_result_count - issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + counts = %i(limited_milestones_count limited_notes_count + limited_merge_requests_count limited_issues_count + blobs_count wiki_blobs_count) + counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend end private @@ -106,7 +117,11 @@ module Gitlab end def notes - @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') + @notes ||= notes_finder(nil) + end + + def notes_finder(type) + NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 781783f4d97..757ef71b95a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -62,22 +62,6 @@ module Gitlab without_count ? collection.without_count : collection end - def projects_count - @projects_count ||= projects.count - end - - def issues_count - @issues_count ||= issues.count - end - - def merge_requests_count - @merge_requests_count ||= merge_requests.count - end - - def milestones_count - @milestones_count ||= milestones.count - end - def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb new file mode 100644 index 00000000000..9a2219b7d77 --- /dev/null +++ b/lib/gitlab/string_placeholder_replacer.rb @@ -0,0 +1,27 @@ +module Gitlab + class StringPlaceholderReplacer + # This method accepts the following paras + # - string: the string to be analyzed + # - placeholder_regex: i.e. /%{project_path|project_id|default_branch|commit_sha}/ + # - block: this block will be called with each placeholder found in the string using + # the placeholder regex. If the result of the block is nil, the original + # placeholder will be returned. + + def self.replace_string_placeholders(string, placeholder_regex = nil, &block) + return string if string.blank? || placeholder_regex.blank? || !block_given? + + replace_placeholders(string, placeholder_regex, &block) + end + + class << self + private + + # If the result of the block is nil, then the placeholder is returned + def replace_placeholders(string, placeholder_regex, &block) + string.gsub(/%{(#{placeholder_regex})}/) do |arg| + yield($~[1]) || arg + end + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index f9faa134206..c6ad997a4d4 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -14,7 +14,7 @@ module Gitlab end def mark(marker_ranges) - return rich_line unless marker_ranges + return rich_line unless marker_ranges&.any? if html_escaped rich_marker_ranges = [] diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index 7ebf1c0428c..b19aa6dea35 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,13 +1,15 @@ module Gitlab class StringRegexMarker < StringRangeMarker def mark(regex, group: 0, &block) - regex_match = raw_line.match(regex) - return rich_line unless regex_match + ranges = [] - begin_index, end_index = regex_match.offset(group) - name_range = begin_index..(end_index - 1) + raw_line.scan(regex) do + begin_index, end_index = Regexp.last_match.offset(group) - super([name_range], &block) + ranges << (begin_index..(end_index - 1)) + end + + super(ranges, &block) end end end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb new file mode 100644 index 00000000000..1ef369a4b67 --- /dev/null +++ b/lib/gitlab/verify/batch_verifier.rb @@ -0,0 +1,64 @@ +module Gitlab + module Verify + class BatchVerifier + attr_reader :batch_size, :start, :finish + + def initialize(batch_size:, start: nil, finish: nil) + @batch_size = batch_size + @start = start + @finish = finish + end + + # Yields a Range of IDs and a Hash of failed verifications (object => error) + def run_batches(&blk) + relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches + range = relation.first.id..relation.last.id + failures = run_batch(relation) + + yield(range, failures) + end + end + + def name + raise NotImplementedError.new + end + + def describe(_object) + raise NotImplementedError.new + end + + private + + def run_batch(relation) + relation.map { |upload| verify(upload) }.compact.to_h + end + + def verify(object) + expected = expected_checksum(object) + actual = actual_checksum(object) + + raise 'Checksum missing' unless expected.present? + raise 'Checksum mismatch' unless expected == actual + + nil + rescue => err + [object, err] + end + + # This should return an ActiveRecord::Relation suitable for calling #in_batches on + def relation + raise NotImplementedError.new + end + + # The checksum we expect the object to have + def expected_checksum(_object) + raise NotImplementedError.new + end + + # The freshly-recalculated checksum of the object + def actual_checksum(_object) + raise NotImplementedError.new + end + end + end +end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb new file mode 100644 index 00000000000..fe51edbdeeb --- /dev/null +++ b/lib/gitlab/verify/lfs_objects.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class LfsObjects < BatchVerifier + def name + 'LFS objects' + end + + def describe(object) + "LFS object: #{object.oid}" + end + + private + + def relation + LfsObject.all + end + + def expected_checksum(lfs_object) + lfs_object.oid + end + + def actual_checksum(lfs_object) + LfsObject.calculate_oid(lfs_object.file.path) + end + end + end +end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb new file mode 100644 index 00000000000..dd138e6b92b --- /dev/null +++ b/lib/gitlab/verify/rake_task.rb @@ -0,0 +1,53 @@ +module Gitlab + module Verify + class RakeTask + def self.run!(verify_kls) + verifier = verify_kls.new( + batch_size: ENV.fetch('BATCH', 200).to_i, + start: ENV['ID_FROM'], + finish: ENV['ID_TO'] + ) + + verbose = Gitlab::Utils.to_boolean(ENV['VERBOSE']) + + new(verifier, verbose).run! + end + + attr_reader :verifier, :output + + def initialize(verifier, verbose) + @verifier = verifier + @verbose = verbose + end + + def run! + say "Checking integrity of #{verifier.name}" + + verifier.run_batches { |*args| run_batch(*args) } + + say 'Done!' + end + + def verbose? + !!@verbose + end + + private + + def say(text) + puts(text) # rubocop:disable Rails/Output + end + + def run_batch(range, failures) + status_color = failures.empty? ? :green : :red + say "- #{range}: Failures: #{failures.count}".color(status_color) + + return unless verbose? + + failures.each do |object, error| + say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + end + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb new file mode 100644 index 00000000000..6972e517ea5 --- /dev/null +++ b/lib/gitlab/verify/uploads.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class Uploads < BatchVerifier + def name + 'Uploads' + end + + def describe(object) + "Upload: #{object.id}" + end + + private + + def relation + Upload.all + end + + def expected_checksum(upload) + upload.checksum + end + + def actual_checksum(upload) + Upload.hexdigest(upload.absolute_path) + end + end + end +end diff --git a/lib/tasks/gitlab/lfs/check.rake b/lib/tasks/gitlab/lfs/check.rake new file mode 100644 index 00000000000..869463d4e5d --- /dev/null +++ b/lib/tasks/gitlab/lfs/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :lfs do + desc 'GitLab | LFS | Check integrity of uploaded LFS objects' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::LfsObjects) + end + end +end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake new file mode 100644 index 00000000000..fd2a4f2d11a --- /dev/null +++ b/lib/tasks/gitlab/traces.rake @@ -0,0 +1,24 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Archive legacy traces to trace artifacts" +namespace :gitlab do + namespace :traces do + task archive: :environment do + logger = Logger.new(STDOUT) + logger.info('Archiving legacy traces') + + Ci::Build.finished + .where('NOT EXISTS (?)', + Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + .order(id: :asc) + .find_in_batches(batch_size: 1000) do |jobs| + job_ids = jobs.map { |job| [job.id] } + + ArchiveTraceWorker.bulk_perform_async(job_ids) + + logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") + end + end + end +end diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake deleted file mode 100644 index df31567ce64..00000000000 --- a/lib/tasks/gitlab/uploads.rake +++ /dev/null @@ -1,44 +0,0 @@ -namespace :gitlab do - namespace :uploads do - desc 'GitLab | Uploads | Check integrity of uploaded files' - task check: :environment do - puts 'Checking integrity of uploaded files' - - uploads_batches do |batch| - batch.each do |upload| - puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) - - if upload.exist? - check_checksum(upload) - else - puts " * File does not exist on the file system".color(:red) - end - end - end - - puts 'Done!' - end - - def batch_size - ENV.fetch('BATCH', 200).to_i - end - - def calculate_checksum(absolute_path) - Digest::SHA256.file(absolute_path).hexdigest - end - - def check_checksum(upload) - checksum = calculate_checksum(upload.absolute_path) - - if checksum != upload.checksum - puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) - end - end - - def uploads_batches(&block) - Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches - yield relation - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/check.rake b/lib/tasks/gitlab/uploads/check.rake new file mode 100644 index 00000000000..2be2ec7f9c9 --- /dev/null +++ b/lib/tasks/gitlab/uploads/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::Uploads) + end + end +end |