diff options
Diffstat (limited to 'lib')
87 files changed, 2116 insertions, 1218 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 94df543853b..79e55a2f4f7 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -2,6 +2,16 @@ module API class API < Grape::API include APIGuard + LOG_FILENAME = Rails.root.join("log", "api_json.log") + + use GrapeLogging::Middleware::RequestLogger, + logger: Logger.new(LOG_FILENAME), + formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, + include: [ + GrapeLogging::Loggers::FilterParameters.new, + GrapeLogging::Loggers::ClientEnv.new + ] + allow_access_with_scope :api prefix :api @@ -108,6 +118,7 @@ module API mount ::API::Internal mount ::API::Issues mount ::API::Jobs + mount ::API::JobArtifacts mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -143,6 +154,7 @@ module API mount ::API::Variables mount ::API::GroupVariables mount ::API::Version + mount ::API::Wikis route :any, '*path' do error!('404 Not Found', 404) diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 0b45621ce7b..d7138b2f2fe 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -20,7 +20,7 @@ module API use :pagination end get do - messages = BroadcastMessage.all + messages = BroadcastMessage.all.order_id_desc present paginate(messages), with: Entities::BroadcastMessage end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index ea78737288a..4b8d248f5f7 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -104,7 +104,7 @@ module API not_found! 'Commit' unless commit - commit.raw_diffs.to_a + present commit.raw_diffs.to_a, with: Entities::RepoDiff end desc "Get a commit's comments" do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 031dd02c6eb..52c49e5caa9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,5 +1,15 @@ module API module Entities + class WikiPageBasic < Grape::Entity + expose :format + expose :slug + expose :title + end + + class WikiPage < WikiPageBasic + expose :content + end + class UserSafe < Grape::Entity expose :id, :name, :username end @@ -35,7 +45,7 @@ module API expose :confirmed_at expose :last_activity_on expose :email - expose :color_scheme_id, :projects_limit, :current_sign_in_at + expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project @@ -119,6 +129,7 @@ module API expose :archived?, as: :archived expose :visibility expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible @@ -290,10 +301,11 @@ module API end class RepoDiff < Grape::Entity - expose :old_path, :new_path, :a_mode, :b_mode, :diff + expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file expose :deleted_file?, as: :deleted_file + expose :json_safe_diff, as: :diff end class ProtectedRefAccess < Grape::Entity @@ -545,7 +557,7 @@ module API end class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_iid, :target_type, :author_id expose :target_title expose :created_at diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 31a918eda60..e817dcbbc4b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -74,7 +74,12 @@ module API use :optional_params end post do - authorize! :create_group + parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? + if parent_group + authorize! :create_subgroup, parent_group + else + authorize! :create_group + end group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3d377fdb9eb..00dbc2aee7a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -56,6 +56,12 @@ module API @project ||= find_project!(params[:id]) end + def wiki_page + page = user_project.wiki.find_page(params[:slug]) + + page || not_found!('Wiki Page') + end + def available_labels @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end @@ -87,7 +93,7 @@ module API end def find_group(id) - if id =~ /^\d+$/ + if id.to_s =~ /^\d+$/ Group.find_by(id: id) else Group.find_by_full_path(id) @@ -128,6 +134,10 @@ module API merge_request end + def find_build!(id) + user_project.builds.find(id.to_i) + end + def authenticate! unauthorized! unless current_user && can?(initial_current_user, :access_api) end @@ -160,6 +170,14 @@ module API authorize! :admin_project, user_project end + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + def require_gitlab_workhorse! unless env['HTTP_GITLAB_WORKHORSE'].present? forbidden!('Request should be executed via GitLab Workhorse') @@ -210,7 +228,7 @@ module API def bad_request!(attribute) message = ["400 (Bad request)"] - message << "\"" + attribute.to_s + "\" not given" + message << "\"" + attribute.to_s + "\" not given" if attribute render_api_error!(message.join(' '), 400) end @@ -432,6 +450,10 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end + def send_artifacts_entry(build, entry) + header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) + end + # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index f57ff0f2632..4c0db4d42b1 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -46,6 +46,15 @@ module API ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end + def redis_ping + result = Gitlab::Redis::SharedState.with { |redis| redis.ping } + + result == 'PONG' + rescue => e + Rails.logger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") + false + end + private def set_project diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 622bd9650e4..c0fef56378f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -88,7 +88,8 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION + gitlab_rev: Gitlab::REVISION, + redis: redis_ping } end @@ -142,6 +143,14 @@ module API { success: true, recovery_codes: codes } end + post '/pre_receive' do + status 200 + + reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase + + { reference_counter_increased: reference_counter_increased } + end + post "/notify_post_receive" do status 200 diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb new file mode 100644 index 00000000000..2a8fa7659bf --- /dev/null +++ b/lib/api/job_artifacts.rb @@ -0,0 +1,80 @@ +module API + class JobArtifacts < Grape::API + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + end + get ':id/jobs/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + desc 'Download the artifacts file from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/artifacts' do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_read_builds! + + build = find_build!(params[:job_id]) + not_found! unless build.artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + bad_request! unless path.valid? + + send_artifacts_entry(build, path) + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/artifacts/keep' do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: Entities::Job + end + end + end +end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 5bab96398fd..3c1c412ba42 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -66,42 +66,11 @@ module API get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) present build, with: Entities::Job end - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/artifacts' do - authorize_read_builds! - - build = get_build!(params[:job_id]) - - present_artifacts!(build.artifacts_file) - end - - desc 'Download the artifacts file from a job' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - end - get ':id/jobs/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_read_builds! - - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) - - present_artifacts!(latest_build.artifacts_file) - end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. @@ -112,7 +81,7 @@ module API get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -131,7 +100,7 @@ module API post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) build.cancel @@ -148,7 +117,7 @@ module API post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? @@ -166,7 +135,7 @@ module API post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? @@ -174,25 +143,6 @@ module API present build, with: Entities::Job end - desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/artifacts/keep' do - authorize_update_builds! - - build = get_build!(params[:job_id]) - authorize!(:update_build, build) - return not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: Entities::Job - end - desc 'Trigger a manual job' do success Entities::Job detail 'This feature was added in GitLab 8.11' @@ -203,7 +153,7 @@ module API post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:job_id]) + build = find_build!(params[:job_id]) authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? @@ -216,14 +166,6 @@ module API end helpers do - def find_build(id) - user_project.builds.find_by(id: id.to_i) - end - - def get_build!(id) - find_build(id) || not_found! - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -234,14 +176,6 @@ module API builds.where(status: available_statuses && scope) end - - def authorize_read_builds! - authorize! :read_build, user_project - end - - def authorize_update_builds! - authorize! :update_build, user_project - end end end end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index ae43a4a3237..d202eaa4c49 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -6,7 +6,7 @@ module API requires :content, type: String, desc: 'Content of .gitlab-ci.yml' end post '/lint' do - error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) status 200 diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index c3affcc6c6b..95ef8f42954 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -21,7 +21,7 @@ module API get ":id/merge_requests/:merge_request_iid/versions" do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff + present paginate(merge_request.merge_request_diffs.order_id_desc), with: Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index ef09d9505d2..c570eace862 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -28,7 +28,7 @@ module API end def list_milestones_for(parent) - milestones = parent.milestones + milestones = parent.milestones.order_id_desc milestones = Milestone.filter_by_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? milestones = filter_by_search(milestones, params[:search]) if params[:search] diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 4845242a173..7dc19788462 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -16,6 +16,7 @@ module API optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' @@ -236,6 +237,7 @@ module API at_least_one_of_ce = [ :jobs_enabled, + :resolve_outdated_diff_discussions, :container_registry_enabled, :default_branch, :description, diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index a9a35f2a4bd..c928ce5265b 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -31,7 +31,7 @@ module API end class Event < Grape::Entity - expose :title, :project_id, :action_name + expose :project_id, :action_name expose :target_id, :target_type, :author_id expose :target_title expose :created_at @@ -64,6 +64,7 @@ module API expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :resolve_outdated_diff_discussions expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb index 35f462e907b..22866fc2845 100644 --- a/lib/api/v3/merge_request_diffs.rb +++ b/lib/api/v3/merge_request_diffs.rb @@ -20,7 +20,7 @@ module API get ":id/merge_requests/:merge_request_id/versions" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff + present merge_request.merge_request_diffs.order_id_desc, with: ::API::Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index 4c7061d4939..9be4cf9d22a 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -34,6 +34,7 @@ module API milestones = user_project.milestones milestones = filter_milestones_state(milestones, params[:state]) milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + milestones = milestones.order_id_desc present paginate(milestones), with: ::API::Entities::Milestone end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 449876c10d9..7c260b8d910 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -18,6 +18,7 @@ module API optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' @@ -119,7 +120,7 @@ module API get do authenticate! - present_projects current_user.authorized_projects, + present_projects current_user.authorized_projects.order_id_desc, with: ::API::V3::Entities::ProjectWithAccess end @@ -296,9 +297,9 @@ module API use :optional_params at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, - :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :public, :visibility_level, :public_builds, - :request_access_enabled, :only_allow_merge_if_build_succeeds, + :shared_runners_enabled, :resolve_outdated_diff_discussions, + :container_registry_enabled, :lfs_enabled, :public, :visibility_level, + :public_builds, :request_access_enabled, :only_allow_merge_if_build_succeeds, :only_allow_merge_if_all_discussions_are_resolved, :path, :default_branch end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb new file mode 100644 index 00000000000..b3fc4e876ad --- /dev/null +++ b/lib/api/wikis.rb @@ -0,0 +1,89 @@ +module API + class Wikis < Grape::API + helpers do + params :wiki_page_params do + requires :content, type: String, desc: 'Content of a wiki page' + requires :title, type: String, desc: 'Title of a wiki page' + optional :format, + type: String, + values: ProjectWiki::MARKUPS.values.map(&:to_s), + default: 'markdown', + desc: 'Format of a wiki page. Available formats are markdown, rdoc, and asciidoc' + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Get a list of wiki pages' do + success Entities::WikiPageBasic + end + params do + optional :with_content, type: Boolean, default: false, desc: "Include pages' content" + end + get ':id/wikis' do + authorize! :read_wiki, user_project + + entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic + present user_project.wiki.pages, with: entity + end + + desc 'Get a wiki page' do + success Entities::WikiPage + end + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + get ':id/wikis/:slug' do + authorize! :read_wiki, user_project + + present wiki_page, with: Entities::WikiPage + end + + desc 'Create a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + post ':id/wikis' do + authorize! :create_wiki, user_project + + page = WikiPages::CreateService.new(user_project, current_user, params).execute + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Update a wiki page' do + success Entities::WikiPage + end + params do + use :wiki_page_params + end + put ':id/wikis/:slug' do + authorize! :create_wiki, user_project + + page = WikiPages::UpdateService.new(user_project, current_user, params).execute(wiki_page) + + if page.valid? + present page, with: Entities::WikiPage + else + render_validation_error!(page) + end + end + + desc 'Delete a wiki page' + params do + requires :slug, type: String, desc: 'The slug of a wiki page' + end + delete ':id/wikis/:slug' do + authorize! :admin_wiki, user_project + + status 204 + WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) + end + end + end +end diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb new file mode 100644 index 00000000000..f5ff95e3eb3 --- /dev/null +++ b/lib/banzai/commit_renderer.rb @@ -0,0 +1,11 @@ +module Banzai + module CommitRenderer + ATTRIBUTES = [:description, :title].freeze + + def self.render(commits, project, user = nil) + obj_renderer = ObjectRenderer.new(project, user) + + ATTRIBUTES.each { |attr| obj_renderer.render(commits, attr) } + end + end +end diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index bcb4f332267..4cd9b02b76c 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -1,6 +1,7 @@ module Banzai module Filter - # HTML filter that moves the value of the src attribute to the data-src attribute so it can be lazy loaded + # HTML filter that moves the value of image `src` attributes to `data-src` + # so they can be lazy loaded. class ImageLazyLoadFilter < HTML::Pipeline::Filter def call doc.xpath('descendant-or-self::img').each do |img| diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 2d6e8ffc90f..9923ec4e870 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -5,6 +5,7 @@ module Banzai # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ def whitelist whitelist = super @@ -24,7 +25,8 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow table alignment + # Allow table alignment; we whitelist specific style properties in a + # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -43,6 +45,9 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) + # Disallow `name` attribute globally + whitelist[:attributes][:all].delete('name') + # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') @@ -52,6 +57,9 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) + # Remove any `style` properties not required for table alignment + whitelist[:transformers].push(self.class.remove_unsafe_table_style) + whitelist end @@ -81,6 +89,21 @@ module Banzai end end end + + def remove_unsafe_table_style + lambda do |env| + node = env[:node] + + return unless node.name == 'th' || node.name == 'td' + return unless node.has_attribute?('style') + + if node['style'] =~ TABLE_ALIGNMENT_PATTERN + node['style'] = "text-align: #{$~[:alignment]}" + else + node.remove_attribute('style') + end + end + end end end end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 8e7084f2543..47151626208 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -22,40 +22,94 @@ module Banzai result[:toc] = "" headers = Hash.new(0) + header_root = current_header = HeaderNode.new doc.css('h1, h2, h3, h4, h5, h6').each do |node| - text = node.text + if header_content = node.children.first + id = node + .text + .downcase + .gsub(PUNCTUATION_REGEXP, '') # remove punctuation + .tr(' ', '-') # replace spaces with dash + .squeeze('-') # replace multiple dashes with one - id = text.downcase - id.gsub!(PUNCTUATION_REGEXP, '') # remove punctuation - id.tr!(' ', '-') # replace spaces with dash - id.squeeze!('-') # replace multiple dashes with one + uniq = headers[id] > 0 ? "-#{headers[id]}" : '' + headers[id] += 1 + href = "#{id}#{uniq}" - uniq = (headers[id] > 0) ? "-#{headers[id]}" : '' - headers[id] += 1 + current_header = HeaderNode.new(node: node, href: href, previous_header: current_header) - if header_content = node.children.first - # namespace detection will be automatically handled via javascript (see issue #22781) - namespace = "user-content-" - href = "#{id}#{uniq}" - push_toc(href, text) - header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) + header_content.add_previous_sibling(anchor_tag(href)) end end - result[:toc] = %Q{<ul class="section-nav">\n#{result[:toc]}</ul>} unless result[:toc].empty? + push_toc(header_root.children, root: true) doc end private - def anchor_tag(id, href) - %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>} + def anchor_tag(href) + %Q{<a id="user-content-#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} end - def push_toc(href, text) - result[:toc] << %Q{<li><a href="##{href}">#{text}</a></li>\n} + def push_toc(children, root: false) + return if children.empty? + + klass = ' class="section-nav"' if root + + result[:toc] << "<ul#{klass}>" + children.each { |child| push_anchor(child) } + result[:toc] << '</ul>' + end + + def push_anchor(header_node) + result[:toc] << %Q{<li><a href="##{header_node.href}">#{header_node.text}</a>} + push_toc(header_node.children) + result[:toc] << '</li>' + end + + class HeaderNode + attr_reader :node, :href, :parent, :children + + def initialize(node: nil, href: nil, previous_header: nil) + @node = node + @href = href + @children = [] + + @parent = find_parent(previous_header) + @parent.children.push(self) if @parent + end + + def level + return 0 unless node + + @level ||= node.name[1].to_i + end + + def text + return '' unless node + + @text ||= node.text + end + + private + + def find_parent(previous_header) + return unless previous_header + + if level == previous_header.level + parent = previous_header.parent + elsif level > previous_header.level + parent = previous_header + else + parent = previous_header + parent = parent.parent while parent.level >= level + end + + parent + end end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 2196a92474c..e40556e869c 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -38,7 +38,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend - object.user_visible_reference_count = redacted_data[:visible_reference_count] + object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) end end diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index e47c384afc1..8f5f144d582 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,6 +1,12 @@ module Banzai module Pipeline class EmailPipeline < FullPipeline + def self.filters + super.tap do |filter_array| + filter_array.delete(Banzai::Filter::ImageLazyLoadFilter) + end + end + def self.transform_context(context) super(context).merge( only_path: false diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 95d82d17658..ceca9296851 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -36,6 +36,10 @@ module Banzai # The context to use is managed by the object and cannot be changed. # Use #render, passing it the field text, if a custom rendering is needed. def self.render_field(object, field) + unless object.respond_to?(:cached_markdown_fields) + return cacheless_render_field(object, field) + end + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) object.cached_html_for(field) diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb deleted file mode 100644 index b9e9f9f7f4a..00000000000 --- a/lib/ci/ansi2html.rb +++ /dev/null @@ -1,331 +0,0 @@ -# ANSI color library -# -# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code -module Ci - module Ansi2html - # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) - COLOR = { - 0 => 'black', # not that this is gray in the intense color table - 1 => 'red', - 2 => 'green', - 3 => 'yellow', - 4 => 'blue', - 5 => 'magenta', - 6 => 'cyan', - 7 => 'white', # not that this is gray in the dark (aka default) color table - }.freeze - - STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 - }.freeze - - def self.convert(ansi, state = nil) - Converter.new.convert(ansi, state) - end - - class Converter - def on_0(s) reset() end - - def on_1(s) enable(STYLE_SWITCHES[:bold]) end - - def on_3(s) enable(STYLE_SWITCHES[:italic]) end - - def on_4(s) enable(STYLE_SWITCHES[:underline]) end - - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end - - def on_9(s) enable(STYLE_SWITCHES[:cross]) end - - def on_21(s) disable(STYLE_SWITCHES[:bold]) end - - def on_22(s) disable(STYLE_SWITCHES[:bold]) end - - def on_23(s) disable(STYLE_SWITCHES[:italic]) end - - def on_24(s) disable(STYLE_SWITCHES[:underline]) end - - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end - - def on_29(s) disable(STYLE_SWITCHES[:cross]) end - - def on_30(s) set_fg_color(0) end - - def on_31(s) set_fg_color(1) end - - def on_32(s) set_fg_color(2) end - - def on_33(s) set_fg_color(3) end - - def on_34(s) set_fg_color(4) end - - def on_35(s) set_fg_color(5) end - - def on_36(s) set_fg_color(6) end - - def on_37(s) set_fg_color(7) end - - def on_38(s) set_fg_color_256(s) end - - def on_39(s) set_fg_color(9) end - - def on_40(s) set_bg_color(0) end - - def on_41(s) set_bg_color(1) end - - def on_42(s) set_bg_color(2) end - - def on_43(s) set_bg_color(3) end - - def on_44(s) set_bg_color(4) end - - def on_45(s) set_bg_color(5) end - - def on_46(s) set_bg_color(6) end - - def on_47(s) set_bg_color(7) end - - def on_48(s) set_bg_color_256(s) end - - def on_49(s) set_bg_color(9) end - - def on_90(s) set_fg_color(0, 'l') end - - def on_91(s) set_fg_color(1, 'l') end - - def on_92(s) set_fg_color(2, 'l') end - - def on_93(s) set_fg_color(3, 'l') end - - def on_94(s) set_fg_color(4, 'l') end - - def on_95(s) set_fg_color(5, 'l') end - - def on_96(s) set_fg_color(6, 'l') end - - def on_97(s) set_fg_color(7, 'l') end - - def on_99(s) set_fg_color(9, 'l') end - - def on_100(s) set_bg_color(0, 'l') end - - def on_101(s) set_bg_color(1, 'l') end - - def on_102(s) set_bg_color(2, 'l') end - - def on_103(s) set_bg_color(3, 'l') end - - def on_104(s) set_bg_color(4, 'l') end - - def on_105(s) set_bg_color(5, 'l') end - - def on_106(s) set_bg_color(6, 'l') end - - def on_107(s) set_bg_color(7, 'l') end - - def on_109(s) set_bg_color(9, 'l') end - - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - - def convert(stream, new_state) - reset_state - restore_state(new_state, stream) if new_state.present? - - append = false - truncated = false - - cur_offset = stream.tell - if cur_offset > @offset - @offset = cur_offset - truncated = true - else - stream.seek(@offset) - append = @offset > 0 - end - start_offset = @offset - - open_new_tag - - stream.each_line do |line| - s = StringScanner.new(line) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) - end - @offset += s.matched_size - end - end - - close_open_tags() - - OpenStruct.new( - html: @out.force_encoding(Encoding.default_external), - state: state, - append: append, - truncated: truncated, - offset: start_offset, - size: stream.tell - start_offset, - total: stream.size - ) - end - - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] - - # We are only interested in color and text style changes - triggered by - # sequences starting with '\e[' and ending with 'm'. Any other control - # sequence gets stripped (including stuff like "delete last line") - return unless indicator == '[' && terminator == 'm' - - close_open_tags() - - if commands.empty?() - reset() - return - end - - evaluate_command_stack(commands) - - open_new_tag - end - - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend - end - - evaluate_command_stack(stack) - end - - def open_new_tag - css_classes = [] - - unless @fg_color.nil? - fg_color = @fg_color - # Most terminals show bold colored text in the light color variant - # Let's mimic that here - if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') - end - css_classes << fg_color - end - css_classes << @bg_color unless @bg_color.nil? - - STYLE_SWITCHES.each do |css_class, flag| - css_classes << "term-#{css_class}" if @style_mask & flag != 0 - end - - return if css_classes.empty? - - @out << %{<span class="#{css_classes.join(' ')}">} - @n_open_tags += 1 - end - - def close_open_tags - while @n_open_tags > 0 - @out << %{</span>} - @n_open_tags -= 1 - end - end - - def reset_state - @offset = 0 - @n_open_tags = 0 - @out = '' - reset - end - - def state - state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend - h - end - Base64.urlsafe_encode64(state.to_json) - end - - def restore_state(new_state, stream) - state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > stream.size - - STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def reset - @fg_color = nil - @bg_color = nil - @style_mask = 0 - end - - def enable(flag) - @style_mask |= flag - end - - def disable(flag) - @style_mask &= ~flag - end - - def set_fg_color(color_index, prefix = nil) - @fg_color = get_term_color_class(color_index, ["fg", prefix]) - end - - def set_bg_color(color_index, prefix = nil) - @bg_color = get_term_color_class(color_index, ["bg", prefix]) - end - - def get_term_color_class(color_index, prefix) - color_name = COLOR[color_index] - return nil if color_name.nil? - - get_color_class(["term", prefix, color_name]) - end - - def set_fg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "fg") - @fg_color = css_class unless css_class.nil? - end - - def set_bg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "bg") - @bg_color = css_class unless css_class.nil? - end - - def get_xterm_color_class(command_stack, prefix) - # the 38 and 48 commands have to be followed by "5" and the color index - return unless command_stack.length >= 2 - return unless command_stack[0] == "5" - - command_stack.shift() # ignore the "5" command - color_index = command_stack.shift().to_i - - return unless color_index >= 0 - return unless color_index <= 255 - - get_color_class(["xterm", prefix, color_index]) - end - - def get_color_class(segments) - [segments].flatten.compact.join('-') - end - end - end -end diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/ci/assets/.gitkeep +++ /dev/null diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb deleted file mode 100644 index 76a69bf8a83..00000000000 --- a/lib/ci/charts.rb +++ /dev/null @@ -1,116 +0,0 @@ -module Ci - module Charts - module DailyInterval - def grouped_count(query) - query - .group("DATE(#{Ci::Pipeline.table_name}.created_at)") - .count(:created_at) - .transform_keys { |date| date.strftime(@format) } - end - - def interval_step - @interval_step ||= 1.day - end - end - - module MonthlyInterval - def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end - end - - def interval_step - @interval_step ||= 1.month - end - end - - class Chart - attr_reader :labels, :total, :success, :project, :pipeline_times - - def initialize(project) - @labels = [] - @total = [] - @success = [] - @pipeline_times = [] - @project = project - - collect - end - - def collect - query = project.pipelines - .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection - - totals_count = grouped_count(query) - success_count = grouped_count(query.success) - - current = @from - while current < @to - label = current.strftime(@format) - - @labels << label - @total << (totals_count[label] || 0) - @success << (success_count[label] || 0) - - current += interval_step - end - end - end - - class YearChart < Chart - include MonthlyInterval - - def initialize(*) - @to = Date.today.end_of_month - @from = @to.years_ago(1).beginning_of_month - @format = '%d %B %Y' - - super - end - end - - class MonthChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 30.days - @format = '%d %B' - - super - end - end - - class WeekChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 7.days - @format = '%d %B' - - super - end - end - - class PipelineTime < Chart - def collect - commits = project.pipelines.last(30) - - commits.each do |commit| - @labels << commit.short_sha - duration = commit.duration || 0 - @pipeline_times << (duration / 60) - end - end - end - end -end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb deleted file mode 100644 index 62b44389b15..00000000000 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ /dev/null @@ -1,251 +0,0 @@ -module Ci - class GitlabCiYamlProcessor - ValidationError = Class.new(StandardError) - - include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - - attr_reader :path, :cache, :stages, :jobs - - def initialize(config, path = nil) - @ci_config = Gitlab::Ci::Config.new(config) - @config = @ci_config.to_hash - @path = path - - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end - - initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e - raise ValidationError, e.message - end - - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| - build_attributes(name) - end - end - - def builds - @jobs.map do |name, _| - build_attributes(name) - end - end - - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) - - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end - - seeds.compact - end - - def build_attributes(name) - job = @jobs[name.to_sym] || {} - - { stage_idx: @stages.index(job[:stage]), - stage: job[:stage], - commands: job[:commands], - tag_list: job[:tags] || [], - name: job[:name].to_s, - allow_failure: job[:ignore], - when: job[:when] || 'on_success', - environment: job[:environment_name], - coverage_regex: job[:coverage], - yaml_variables: yaml_variables(name), - options: { - image: job[:image], - services: job[:services], - artifacts: job[:artifacts], - cache: job[:cache], - dependencies: job[:dependencies], - before_script: job[:before_script], - script: job[:script], - after_script: job[:after_script], - environment: job[:environment], - retry: job[:retry] - }.compact } - end - - def self.validation_message(content) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Ci::GitlabCiYamlProcessor.new(content) - nil - rescue ValidationError, Psych::SyntaxError => e - e.message - end - end - - private - - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_active? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def initial_parsing - ## - # Global config - # - @before_script = @ci_config.before_script - @image = @ci_config.image - @after_script = @ci_config.after_script - @services = @ci_config.services - @variables = @ci_config.variables - @stages = @ci_config.stages - @cache = @ci_config.cache - - ## - # Jobs - # - @jobs = @ci_config.jobs - - @jobs.each do |name, job| - # logical validation for job - - validate_job_stage!(name, job) - validate_job_dependencies!(name, job) - validate_job_environment!(name, job) - end - end - - def yaml_variables(name) - variables = (@variables || {}) - .merge(job_variables(name)) - - variables.map do |key, value| - { key: key.to_s, value: value, public: true } - end - end - - def job_variables(name) - job = @jobs[name.to_sym] - return {} unless job - - job[:variables] || {} - end - - def validate_job_stage!(name, job) - return unless job[:stage] - - unless job[:stage].is_a?(String) && job[:stage].in?(@stages) - raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" - end - end - - def validate_job_dependencies!(name, job) - return unless job[:dependencies] - - stage_index = @stages.index(job[:stage]) - - job[:dependencies].each do |dependency| - raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index - raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" - end - end - end - - def validate_job_environment!(name, job) - return unless job[:environment] - return unless job[:environment].is_a?(Hash) - - environment = job[:environment] - validate_on_stop_job!(name, environment, environment[:on_stop]) - end - - def validate_on_stop_job!(name, environment, on_stop) - return unless on_stop - - on_stop_job = @jobs[on_stop.to_sym] - unless on_stop_job - raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" - end - - unless on_stop_job[:environment] - raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" - end - - unless on_stop_job[:environment][:name] == environment[:name] - raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" - end - - unless on_stop_job[:environment][:action] == 'stop' - raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" - end - end - - def process?(only_params, except_params, ref, tag, source) - if only_params.present? - return false unless matching?(only_params, ref, tag, source) - end - - if except_params.present? - return false if matching?(except_params, ref, tag, source) - end - - true - end - - def matching?(patterns, ref, tag, source) - patterns.any? do |pattern| - pattern, path = pattern.split('@', 2) - matches_path?(path) && matches_pattern?(pattern, ref, tag, source) - end - end - - def matches_path?(path) - return true unless path - - path == self.path - end - - def matches_pattern?(pattern, ref, tag, source) - return true if tag && pattern == 'tags' - return true if !tag && pattern == 'branches' - return true if source_to_pattern(source) == pattern - - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end - end -end diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb deleted file mode 100644 index 997377abc55..00000000000 --- a/lib/ci/mask_secret.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Ci::MaskSecret - class << self - def mask!(value, token) - return value unless value.present? && token.present? - - value.gsub!(token, 'x' * token.length) - value - end - end -end diff --git a/lib/ci/model.rb b/lib/ci/model.rb deleted file mode 100644 index c42a0ad36db..00000000000 --- a/lib/ci/model.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Ci - module Model - def table_name_prefix - "ci_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) - end - end -end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index c6fa928d565..823e8e9a9c4 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -41,7 +41,7 @@ module Github def remove!(name) repository.delete_branch(name) - rescue Rugged::ReferenceError => e + rescue Gitlab::Git::Repository::DeleteBranchError => e Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 4714ab18cc1..b4012ebbb99 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -67,10 +67,14 @@ module Gitlab def protection_values protection_options.values end + + def human_access(access) + options_with_owner.key(access) + end end def human_access - Gitlab::Access.options_with_owner.key(access_field) + Gitlab::Access.human_access(access_field) end def owner? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3fd81759d25..11ace83c15c 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,7 +2,7 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) - REGISTRY_SCOPES = [:read_registry].freeze + REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb new file mode 100644 index 00000000000..ad78ae244b2 --- /dev/null +++ b/lib/gitlab/ci/ansi2html.rb @@ -0,0 +1,333 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Gitlab + module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + }.freeze + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 + }.freeze + + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) + end + + class Converter + def on_0(s) reset() end + + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + + def on_31(s) set_fg_color(1) end + + def on_32(s) set_fg_color(2) end + + def on_33(s) set_fg_color(3) end + + def on_34(s) set_fg_color(4) end + + def on_35(s) set_fg_color(5) end + + def on_36(s) set_fg_color(6) end + + def on_37(s) set_fg_color(7) end + + def on_38(s) set_fg_color_256(s) end + + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + + def on_41(s) set_bg_color(1) end + + def on_42(s) set_bg_color(2) end + + def on_43(s) set_bg_color(3) end + + def on_44(s) set_bg_color(4) end + + def on_45(s) set_bg_color(5) end + + def on_46(s) set_bg_color(6) end + + def on_47(s) set_bg_color(7) end + + def on_48(s) set_bg_color_256(s) end + + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + + def on_91(s) set_fg_color(1, 'l') end + + def on_92(s) set_fg_color(2, 'l') end + + def on_93(s) set_fg_color(3, 'l') end + + def on_94(s) set_fg_color(4, 'l') end + + def on_95(s) set_fg_color(5, 'l') end + + def on_96(s) set_fg_color(6, 'l') end + + def on_97(s) set_fg_color(7, 'l') end + + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + + def on_101(s) set_bg_color(1, 'l') end + + def on_102(s) set_bg_color(2, 'l') end + + def on_103(s) set_bg_color(3, 'l') end + + def on_104(s) set_bg_color(4, 'l') end + + def on_105(s) set_bg_color(5, 'l') end + + def on_106(s) set_bg_color(6, 'l') end + + def on_107(s) set_bg_color(7, 'l') end + + def on_109(s) set_bg_color(9, 'l') end + + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + + def convert(stream, new_state) + reset_state + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset + + open_new_tag + + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size + end + end + + close_open_tags() + + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' && terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend + end + + evaluate_command_stack(stack) + end + + def open_new_tag + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + return if css_classes.empty? + + @out << %{<span class="#{css_classes.join(' ')}">} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{</span>} + @n_open_tags -= 1 + end + end + + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(new_state, stream) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > stream.size + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 2e073334abc..22941d48edf 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,129 +1,129 @@ module Gitlab - module Ci::Build::Artifacts - class Metadata - ## - # Class that represents an entry (path and metadata) to a file or - # directory in GitLab CI Build Artifacts binary file / archive - # - # This is IO-operations safe class, that does similar job to - # Ruby's Pathname but without the risk of accessing filesystem. - # - # This class is working only with UTF-8 encoded paths. - # - class Entry - attr_reader :path, :entries - attr_accessor :name - - def initialize(path, entries) - @path = path.dup.force_encoding('UTF-8') - @entries = entries - - if path.include?("\0") - raise ArgumentError, 'Path contains zero byte character!' - end + module Ci + module Build + module Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :entries + attr_accessor :name + + def initialize(path, entries) + @entries = entries + @path = Artifacts::Path.new(path) + end + + delegate :empty?, to: :children + + def directory? + blank_node? || @path.directory? + end + + def file? + !directory? + end + + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.name + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path.to_s)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path.to_s] || {} + end + + def nodes + @path.nodes + (file? ? 1 : 0) + end + + def blank_node? + @path.to_s.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path.to_s) + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def path + @path.to_s + end + + def to_s + @path.to_s + end + + def ==(other) + path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{self}" + end - unless path.valid_encoding? - raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end end end - - delegate :empty?, to: :children - - def directory? - blank_node? || @path.end_with?('/') - end - - def file? - !directory? - end - - def blob - return unless file? - - @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) - end - - def has_parent? - nodes > 0 - end - - def parent - return nil unless has_parent? - self.class.new(@path.chomp(basename), @entries) - end - - def basename - (directory? && !blank_node?) ? name + '/' : name - end - - def name - @name || @path.split('/').last.to_s - end - - def children - return [] unless directory? - return @children if @children - - child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} - @children = select_entries { |path| path =~ child_pattern } - end - - def directories(opts = {}) - return [] unless directory? - dirs = children.select(&:directory?) - return dirs unless has_parent? && opts[:parent] - - dotted_parent = parent - dotted_parent.name = '..' - dirs.prepend(dotted_parent) - end - - def files - return [] unless directory? - children.select(&:file?) - end - - def metadata - @entries[@path] || {} - end - - def nodes - @path.count('/') + (file? ? 1 : 0) - end - - def blank_node? - @path.empty? # "" is considered to be './' - end - - def exists? - blank_node? || @entries.include?(@path) - end - - def total_size - descendant_pattern = %r{^#{Regexp.escape(@path)}} - entries.sum do |path, entry| - (entry[:size] if path =~ descendant_pattern).to_i - end - end - - def to_s - @path - end - - def ==(other) - @path == other.path && @entries == other.entries - end - - def inspect - "#{self.class.name}: #{@path}" - end - - private - - def select_entries - selected = @entries.select { |path, _metadata| yield path } - selected.map { |path, _metadata| self.class.new(path, @entries) } - end end end end diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb new file mode 100644 index 00000000000..9cd9b36c5f8 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -0,0 +1,51 @@ +module Gitlab + module Ci + module Build + module Artifacts + class Path + def initialize(path) + @path = path.dup.force_encoding('UTF-8') + end + + def valid? + nonzero? && utf8? + end + + def directory? + @path.end_with?('/') + end + + def name + @path.split('/').last.to_s + end + + def nodes + @path.count('/') + end + + def to_s + @path.tap do |path| + unless nonzero? + raise ArgumentError, 'Path contains zero byte character!' + end + + unless utf8? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + end + + private + + def nonzero? + @path.exclude?("\0") + end + + def utf8? + @path.valid_encoding? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb new file mode 100644 index 00000000000..7df7b542d91 --- /dev/null +++ b/lib/gitlab/ci/charts.rb @@ -0,0 +1,118 @@ +module Gitlab + module Ci + module Charts + module DailyInterval + def grouped_count(query) + query + .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + .transform_keys { |date| date.strftime(@format) } + end + + def interval_step + @interval_step ||= 1.day + end + end + + module MonthlyInterval + def grouped_count(query) + if Gitlab::Database.postgresql? + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) + else + query + .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") + .count(:created_at) + end + end + + def interval_step + @interval_step ||= 1.month + end + end + + class Chart + attr_reader :labels, :total, :success, :project, :pipeline_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @pipeline_times = [] + @project = project + + collect + end + + def collect + query = project.pipelines + .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + + totals_count = grouped_count(query) + success_count = grouped_count(query.success) + + current = @from + while current < @to + label = current.strftime(@format) + + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) + + current += interval_step + end + end + end + + class YearChart < Chart + include MonthlyInterval + + def initialize(*) + @to = Date.today.end_of_month + @from = @to.years_ago(1).beginning_of_month + @format = '%d %B %Y' + + super + end + end + + class MonthChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 30.days + @format = '%d %B' + + super + end + end + + class WeekChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 7.days + @format = '%d %B' + + super + end + end + + class PipelineTime < Chart + def collect + commits = project.pipelines.last(30) + + commits.each do |commit| + @labels << commit.short_sha + duration = commit.duration || 0 + @pipeline_times << (duration / 60) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb new file mode 100644 index 00000000000..0daddaa638c --- /dev/null +++ b/lib/gitlab/ci/mask_secret.rb @@ -0,0 +1,12 @@ +module Gitlab + module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end + end +end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb new file mode 100644 index 00000000000..3994a50772b --- /dev/null +++ b/lib/gitlab/ci/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 8503ecf8700..ab3408f48d6 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -56,13 +56,13 @@ module Gitlab end def html_with_state(state = nil) - ::Ci::Ansi2html.convert(stream, state) + ::Gitlab::Ci::Ansi2html.convert(stream, state) end def html(last_lines: nil) text = raw(last_lines: last_lines) buffer = StringIO.new(text) - ::Ci::Ansi2html.convert(buffer).html + ::Gitlab::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb new file mode 100644 index 00000000000..7582964b24e --- /dev/null +++ b/lib/gitlab/ci/yaml_processor.rb @@ -0,0 +1,253 @@ +module Gitlab + module Ci + class YamlProcessor + ValidationError = Class.new(StandardError) + + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + + attr_reader :path, :cache, :stages, :jobs + + def initialize(config, path = nil) + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path + + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + + initial_parsing + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message + end + + def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| + build_attributes(name) + end + end + + def builds + @jobs.map do |name, _| + build_attributes(name) + end + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + + def build_attributes(name) + job = @jobs[name.to_sym] || {} + + { stage_idx: @stages.index(job[:stage]), + stage: job[:stage], + commands: job[:commands], + tag_list: job[:tags] || [], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: yaml_variables(name), + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + cache: job[:cache], + dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry] + }.compact } + end + + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Gitlab::Ci::YamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + + private + + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + + def initial_parsing + ## + # Global config + # + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache + + ## + # Jobs + # + @jobs = @ci_config.jobs + + @jobs.each do |name, job| + # logical validation for job + + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_environment!(name, job) + end + end + + def yaml_variables(name) + variables = (@variables || {}) + .merge(job_variables(name)) + + variables.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job + + job[:variables] || {} + end + + def validate_job_stage!(name, job) + return unless job[:stage] + + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + end + end + + def validate_job_dependencies!(name, job) + return unless job[:dependencies] + + stage_index = @stages.index(job[:stage]) + + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] + + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end + end + end + + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + + def process?(only_params, except_params, ref, tag, source) + if only_params.present? + return false unless matching?(only_params, ref, tag, source) + end + + if except_params.present? + return false if matching?(except_params, ref, tag, source) + end + + true + end + + def matching?(patterns, ref, tag, source) + patterns.any? do |pattern| + pattern, path = pattern.split('@', 2) + matches_path?(path) && matches_pattern?(pattern, ref, tag, source) + end + end + + def matches_path?(path) + return true unless path + + path == self.path + end + + def matches_pattern?(pattern, ref, tag, source) + return true if tag && pattern == 'tags' + return true if !tag && pattern == 'branches' + return true if source_to_pattern(source) == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end + end + + def source_to_pattern(source) + if %w[api external web].include?(source) + source + else + source&.pluralize + end + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 84f9ecd3d23..e3678c914db 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -12,12 +12,7 @@ module Gitlab MissingEndDelimiter = Class.new(ParserError) def parse(text, our_path:, their_path:, parent_file: nil) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? + validate_text!(text) line_obj_index = 0 line_old = 1 @@ -32,15 +27,15 @@ module Gitlab full_line = line.delete("\n") if full_line == conflict_start - raise UnexpectedDelimiter unless type.nil? + validate_delimiter!(type.nil?) type = 'new' elsif full_line == conflict_middle - raise UnexpectedDelimiter unless type == 'new' + validate_delimiter!(type == 'new') type = 'old' elsif full_line == conflict_end - raise UnexpectedDelimiter unless type == 'old' + validate_delimiter!(type == 'old') type = nil elsif line[0] == '\\' @@ -59,6 +54,21 @@ module Gitlab lines end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end end end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 8ddc91e341d..7b3483a7f96 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -22,10 +22,10 @@ module Gitlab # return message if message type is binary detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary + return message.force_encoding("BINARY") if detect_binary?(message, detect) - # force detected encoding if we have sufficient confidence. if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + # force detected encoding if we have sufficient confidence. message.force_encoding(detect[:encoding]) end @@ -36,6 +36,19 @@ module Gitlab "--broken encoding: #{encoding}" end + def detect_binary?(data, detect = nil) + detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect && detect[:type] == :binary && detect[:confidence] == 100 + end + + def detect_libgit2_binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary + end + def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3784f6c4947..3f7b42456af 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -25,6 +25,12 @@ module Gitlab end EOS + def self.get_uuid(key) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_shared_state_key(key)) || false + end + end + def self.cancel(key, uuid) Gitlab::Redis::SharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) @@ -35,10 +41,10 @@ module Gitlab "gitlab:exclusive_lease:#{key}" end - def initialize(key, timeout:) + def initialize(key, uuid: nil, timeout:) @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout - @uuid = SecureRandom.uuid + @uuid = uuid || SecureRandom.uuid end # Try to obtain the lease. Return lease UUID on success, diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8c9acbc9fbe..b4b6326cfdd 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 7780f4e4d4f..8d96826f6ee 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -42,14 +42,6 @@ module Gitlab end end - def binary?(data) - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) - detect && detect[:type] == :binary - end - # Returns an array of Blob instances, specified in blob_references as # [[commit_sha, path], [commit_sha, path], ...]. If blob_size_limit < 0 then the # full blob contents are returned. If blob_size_limit >= 0 then each blob will @@ -65,6 +57,10 @@ module Gitlab end end + def binary?(data) + EncodingHelper.detect_libgit2_binary?(data) + end + private # Recursive search of blob id by path diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5ee6669050c..1f370686186 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -352,7 +352,7 @@ module Gitlab end def stats - Gitlab::Git::CommitStats.new(self) + Gitlab::Git::CommitStats.new(@repository, self) end def to_patch(options = {}) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 00acb4763e9..6bf49a0af18 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -10,12 +10,29 @@ module Gitlab # Instantiate a CommitStats object # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 - def initialize(commit) + def initialize(repo, commit) @id = commit.id @additions = 0 @deletions = 0 @total = 0 + repo.gitaly_migrate(:commit_stats) do |is_enabled| + if is_enabled + gitaly_stats(repo, commit) + else + rugged_stats(commit) + end + end + end + + def gitaly_stats(repo, commit) + stats = repo.gitaly_commit_client.commit_stats(@id) + @additions = stats.additions + @deletions = stats.deletions + @total = @additions + @deletions + end + + def rugged_stats(commit) diff = commit.rugged_diff_from_parent diff.each_patch do |p| diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ce3d65062e8..a23c8cf0dd1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -116,6 +116,15 @@ module Gitlab filtered_opts end + + # Return a binary diff message like: + # + # "Binary files a/file/path and b/file/path differ\n" + # This is used when we detect that a diff is binary + # using CharlockHolmes when Rugged treats it as text. + def binary_message(old_path, new_path) + "Binary files #{old_path} and #{new_path} differ\n" + end end def initialize(raw_diff, expanded: true) @@ -190,6 +199,13 @@ module Gitlab @collapsed = true end + def json_safe_diff + return @diff unless detect_binary?(@diff) + + # the diff is binary, let's make a message for it + Diff.binary_message(@old_path, @new_path) + end + private def init_from_rugged(rugged) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 75d4efc0bc5..efa13590a2c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,7 @@ module Gitlab InvalidBlobName = Class.new(StandardError) InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) + DeleteBranchError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the storage path, not the storage name @@ -653,10 +654,16 @@ module Gitlab end # Delete the specified branch from the repository - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def delete_branch(branch_name) - rugged.branches.delete(branch_name) + gitaly_migrate(:delete_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_branch(branch_name) + else + rugged.branches.delete(branch_name) + end + end + rescue Rugged::ReferenceError, CommandError => e + raise DeleteBranchError, e end def delete_refs(*ref_names) @@ -681,15 +688,14 @@ module Gitlab # Examples: # create_branch("feature") # create_branch("other-feature", "master") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/476 def create_branch(ref, start_point = "HEAD") - rugged_ref = rugged.branches.create(ref, start_point) - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ - raise InvalidRef.new("Invalid reference #{start_point}") + gitaly_migrate(:create_branch) do |is_enabled| + if is_enabled + gitaly_ref_client.create_branch(ref, start_point) + else + rugged_create_branch(ref, start_point) + end + end end # Delete the specified remote from this repository. @@ -1226,6 +1232,15 @@ module Gitlab false end + def rugged_create_branch(ref, start_point) + rugged_ref = rugged.branches.create(ref, start_point) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + def gitaly_copy_gitattributes(revision) gitaly_repository_client.apply_gitattributes(revision) end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b54962a4456..5cf336af3c6 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -5,7 +5,7 @@ module Gitlab class Tree include Gitlab::EncodingHelper - attr_accessor :id, :root_id, :name, :path, :type, + attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url class << self @@ -19,8 +19,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| if is_enabled - client = Gitlab::GitalyClient::CommitService.new(repository) - client.tree_entries(repository, sha, path) + repository.gitaly_commit_client.tree_entries(repository, sha, path) else tree_entries_from_rugged(repository, sha, path) end @@ -88,7 +87,7 @@ module Gitlab end def initialize(options) - %w(id root_id name path type mode commit_id).each do |key| + %w(id root_id name path flat_path type mode commit_id).each do |key| self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end @@ -101,6 +100,10 @@ module Gitlab encode! @path end + def flat_path + encode! @flat_path + end + def dir? type == :tree end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 9a5f4f598b2..a3dc2cd0b60 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -70,21 +70,41 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + # Evaluates whether a feature toggle is on or off + def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN) + # Disabled features are always off! return false if status == MigrationStatus::DISABLED - feature = Feature.get("gitaly_#{feature}") + feature = Feature.get("gitaly_#{feature_name}") - # If the feature hasn't been set, turn it on if it's opt-out - return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + # If the feature has been set, always evaluate + if Feature.persisted?(feature) + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + return feature.enabled? + end - if feature.percentage_of_time_value > 0 - # Probabilistically enable this feature - return Random.rand() * 100 < feature.percentage_of_time_value + # If the feature has not been set, the default depends + # on it's status + case status + when MigrationStatus::OPT_OUT + true + when MigrationStatus::OPT_IN + opt_into_all_features? + else + false end + end - feature.enabled? + # opt_into_all_features? returns true when the current environment + # is one in which we opt into features automatically + def self.opt_into_all_features? + Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1" end + private_class_method :opt_into_all_features? def self.migrate(feature, status: MigrationStatus::OPT_IN) is_enabled = feature_enabled?(feature, status: status) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 21a32a7e0db..1ba1a7830a4 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -88,14 +88,14 @@ module Gitlab response.flat_map do |message| message.entries.map do |gitaly_tree_entry| - entry_path = gitaly_tree_entry.path.dup Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, root_id: gitaly_tree_entry.root_oid, type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), - name: File.basename(entry_path), - path: entry_path, + name: File.basename(gitaly_tree_entry.path), + path: GitalyClient.encode(gitaly_tree_entry.path), + flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -204,6 +204,14 @@ module Gitlab response.sum(&:data) end + def commit_stats(revision) + request = Gitaly::CommitStatsRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + end + private def commit_diff_request_params(commit, options = {}) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index a1a25cf2079..8ef873d5848 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -79,7 +79,7 @@ module Gitlab end def find_branch(branch_name) - request = Gitaly::DeleteBranchRequest.new( + request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, name: GitalyClient.encode(branch_name) ) @@ -92,6 +92,40 @@ module Gitlab Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) end + def create_branch(ref, start_point) + request = Gitaly::CreateBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(ref), + start_point: GitalyClient.encode(start_point) + ) + + response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) + + case response.status + when :OK + branch = response.branch + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) + when :ERR_INVALID + invalid_ref!("Invalid ref name") + when :ERR_EXISTS + invalid_ref!("Branch #{ref} already exists") + when :ERR_INVALID_START_POINT + invalid_ref!("Invalid reference #{start_point}") + else + raise "Unknown response status: #{response.status}" + end + end + + def delete_branch(branch_name) + request = Gitaly::DeleteBranchRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(branch_name) + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) + end + private def consume_refs_response(response) @@ -163,6 +197,10 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + + def invalid_ref!(message) + raise Gitlab::Git::Repository::InvalidRef.new(message) + end end end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 373062b354b..b8c07460ebb 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -166,7 +166,7 @@ module Gitlab def remove_branch(name) project.repository.delete_branch(name) - rescue Rugged::ReferenceError + rescue Gitlab::Git::Repository::DeleteBranchFailed errors << { type: :remove_branch, name: name } end diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 025f826e65f..0d5039ddf5f 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -69,11 +69,17 @@ module Gitlab def optimistic_using_tmp_keychain previous_dir = current_home_dir - Dir.mktmpdir do |dir| - GPGME::Engine.home_dir = dir - yield - end + tmp_dir = Dir.mktmpdir + GPGME::Engine.home_dir = tmp_dir + yield ensure + # Ignore any errors when removing the tmp directory, as we may run into a + # race condition: + # The `gpg-agent` agent process may clean up some files as well while + # `FileUtils.remove_entry` is iterating the directory and removing all + # its contained files and directories recursively, which could raise an + # error. + FileUtils.remove_entry(tmp_dir, true) GPGME::Engine.home_dir = previous_dir end end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb new file mode 100644 index 00000000000..1e1fdabca93 --- /dev/null +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -0,0 +1,19 @@ +module Gitlab + module GrapeLogging + module Formatters + class LogrageWithTimestamp + def call(severity, datetime, _, data) + time = data.delete :time + attributes = { + time: datetime.utc.iso8601(3), + severity: severity, + duration: time[:total], + db: time[:db], + view: time[:view] + }.merge(data) + ::Lograge.formatter.call(attributes) + "\n" + end + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 3470a09eaf0..50ee879129c 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.8'.freeze + VERSION = '0.2.0'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 78795dd3d92..2171c6c7bbb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -50,6 +50,7 @@ project_tree: - :push_event_payload - :stages - :statuses + - :auto_devops - :triggers - :pipeline_schedules - :services @@ -116,6 +117,7 @@ excluded_attributes: statuses: - :trace - :token + - :when push_event_payload: - :event_id @@ -141,4 +143,4 @@ methods: events: - :action push_event_payload: - - :action
\ No newline at end of file + - :action diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index cbc8d170936..3bc095a99a9 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,6 +9,8 @@ module Gitlab @user = user @shared = shared @project = project + @project_id = project.id + @saved = true end def restore @@ -22,8 +24,10 @@ module Gitlab @project_members = @tree_hash.delete('project_members') - ActiveRecord::Base.no_touching do - create_relations + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + create_relations + end end rescue => e @shared.error(e) @@ -48,21 +52,24 @@ module Gitlab # the configuration yaml file too. # Finally, it updates each attribute in the newly imported project. def create_relations - saved = [] default_relation_list.each do |relation| - next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + if relation.is_a?(Hash) + create_sub_relations(relation, @tree_hash) + elsif @tree_hash[relation.to_s].present? + save_relation_hash(@tree_hash[relation.to_s], relation) + end + end - create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + @saved + end - relation_key = relation.is_a?(Hash) ? relation.keys.first : relation - relation_hash_list = @tree_hash[relation_key.to_s] + def save_relation_hash(relation_hash_batch, relation_key) + relation_hash = create_relation(relation_key, relation_hash_batch) - next unless relation_hash_list + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) - relation_hash = create_relation(relation_key, relation_hash_list) - saved << restored_project.append_or_update_attribute(relation_key, relation_hash) - end - saved.all? + # Restore the project again, extra query that skips holding the AR objects in memory + @restored_project = Project.find(@project_id) end def default_relation_list @@ -93,20 +100,42 @@ module Gitlab # issue, finds any subrelations such as notes, creates them and assign them back to the hash # # Recursively calls this method if the sub-relation is a hash containing more sub-relations - def create_sub_relations(relation, tree_hash) + def create_sub_relations(relation, tree_hash, save: true) relation_key = relation.keys.first.to_s return if tree_hash[relation_key].blank? - [tree_hash[relation_key]].flatten.each do |relation_item| - relation.values.flatten.each do |sub_relation| - # We just use author to get the user ID, do not attempt to create an instance. - next if sub_relation == :author + tree_array = [tree_hash[relation_key]].flatten + + # Avoid keeping a possible heavy object in memory once we are done with it + while relation_item = tree_array.shift + # The transaction at this level is less speedy than one single transaction + # But we can't have it in the upper level or GC won't get rid of the AR objects + # after we save the batch. + Project.transaction do + process_sub_relation(relation, relation_item) + + # For every subrelation that hangs from Project, save the associated records alltogether + # This effectively batches all records per subrelation item, only keeping those in memory + # We have to keep in mind that more batch granularity << Memory, but >> Slowness + if save + save_relation_hash([relation_item], relation_key) + tree_hash[relation_key].delete(relation_item) + end + end + end + + tree_hash.delete(relation_key) if save + end + + def process_sub_relation(relation, relation_item) + relation.values.flatten.each do |sub_relation| + # We just use author to get the user ID, do not attempt to create an instance. + next if sub_relation == :author - create_sub_relations(sub_relation, relation_item) if sub_relation.is_a?(Hash) + create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash) - relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) - relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? - end + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? end end @@ -121,14 +150,12 @@ module Gitlab end def create_relation(relation, relation_hash_list) - relation_type = relation.to_sym - relation_array = [relation_hash_list].flatten.map do |relation_hash| - Gitlab::ImportExport::RelationFactory.create(relation_sym: relation_type, - relation_hash: parsed_relation_hash(relation_hash, relation_type), + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), members_mapper: members_mapper, user: @user, - project: restored_project) + project: @restored_project) end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 20580459046..380b336395d 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,6 +14,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', labels: :project_labels, priorities: :label_priorities, + auto_devops: :project_auto_devops, label: :project_label }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze @@ -69,7 +70,6 @@ module Gitlab reset_tokens! remove_encrypted_attributes! - @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff set_diff if @relation_name == :merge_request_diff_files end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 5d6de8bc475..9fd0b709ef2 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -16,7 +16,7 @@ module Gitlab error_out(error.message, caller[0].dup) @errors << error.message # Debug: - Rails.logger.error(error.backtrace) + Rails.logger.error(error.backtrace.join("\n")) end private diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 39180dc17d9..3bf27b37ae6 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -36,7 +36,7 @@ module Gitlab end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) end def update_user_attributes @@ -60,7 +60,7 @@ module Gitlab ldap_config.block_auto_created_users end - def sync_email_from_provider? + def sync_profile_from_provider? true end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 9f432673a6e..344784c866f 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -4,6 +4,15 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom + DEFAULT_CONFIG = { + enabled: false, + port: 143, + ssl: false, + start_tls: false, + mailbox: 'inbox', + idle_timeout: 60 + }.freeze + class << self def enabled? config[:enabled] && config[:address] @@ -22,16 +31,10 @@ module Gitlab def fetch_config return {} unless File.exist?(config_file) - rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' - all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys - - config = all_config[:incoming_email] || {} - config[:enabled] = false if config[:enabled].nil? - config[:port] = 143 if config[:port].nil? - config[:ssl] = false if config[:ssl].nil? - config[:start_tls] = false if config[:start_tls].nil? - config[:mailbox] = 'inbox' if config[:mailbox].nil? - config[:idle_timeout] = 60 if config[:idle_timeout].nil? + config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] || {} + config = DEFAULT_CONFIG.merge(config) do |_key, oldval, newval| + newval.nil? ? oldval : newval + end if config[:enabled] && config[:address] gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) @@ -45,6 +48,10 @@ module Gitlab config end + def rails_env + @rails_env ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + end + def config_file ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 6023fa1820f..f42168c720e 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -3,6 +3,10 @@ module Gitlab module Middleware class Go + include ActionView::Helpers::TagHelper + + PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze + def initialize(app) @app = app end @@ -10,17 +14,20 @@ module Gitlab def call(env) request = Rack::Request.new(env) - if go_request?(request) - render_go_doc(request) - else - @app.call(env) - end + render_go_doc(request) || @app.call(env) end private def render_go_doc(request) - body = go_body(request) + return unless go_request?(request) + + path = project_path(request) + return unless path + + body = go_body(path) + return unless body + response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' }) response.finish end @@ -29,11 +36,13 @@ module Gitlab request["go-get"].to_i == 1 && request.env["PATH_INFO"].present? end - def go_body(request) - project_url = URI.join(Gitlab.config.gitlab.url, project_path(request)) + def go_body(path) + project_url = URI.join(Gitlab.config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n" + meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{project_url}.git" + head_tag = content_tag :head, meta_tag + content_tag :html, head_tag end def strip_url(url) @@ -44,6 +53,10 @@ module Gitlab path_info = request.env["PATH_INFO"] path_info.sub!(/^\//, '') + project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) + return unless project_path_match + path = project_path_match[1] + # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. # In a traditional project with a single namespace, this would denote repo # `namespace/project` with subpath `path1/path2/../pathN`, but with nested @@ -51,7 +64,7 @@ module Gitlab # `path2/../pathN`, for example. # We find all potential project paths out of the path segments - path_segments = path_info.split('/') + path_segments = path.split('/') simple_project_path = path_segments.first(2).join('/') # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 7d6911a1ab3..1f331b1e91d 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,8 +32,21 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end - def has_email? - get_info(:email).present? + def location + location = get_info(:address) + if location.is_a?(Hash) + [location.locality.presence, location.country.presence].compact.join(', ') + else + location + end + end + + def has_attribute?(attribute) + if attribute == :location + get_info(:address).present? + else + get_info(attribute).present? + end end private diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e8330917e91..7704bf715e4 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash - update_email + update_profile if sync_profile_from_provider? end def persisted? @@ -184,20 +184,30 @@ module Gitlab } end - def sync_email_from_provider? - auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + def sync_profile_from_provider? + providers = Gitlab.config.omniauth.sync_profile_from_provider + + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end - def update_email - if auth_hash.has_email? && sync_email_from_provider? - if persisted? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end + def update_profile + user_synced_attributes_metadata = gl_user.user_synced_attributes_metadata || gl_user.build_user_synced_attributes_metadata - gl_user.external_email = true - gl_user.email_provider = auth_hash.provider + UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| + if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) + gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + user_synced_attributes_metadata.set_attribute_synced(key, true) + else + user_synced_attributes_metadata.set_attribute_synced(key, false) + end end + + user_synced_attributes_metadata.provider = auth_hash.provider + gl_user.user_synced_attributes_metadata = user_synced_attributes_metadata end def log diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb new file mode 100644 index 00000000000..981ef8faa9a --- /dev/null +++ b/lib/gitlab/pages.rb @@ -0,0 +1,5 @@ +module Gitlab + module Pages + VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 894bd5efae5..7c02c9c5c48 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -26,6 +26,7 @@ module Gitlab apple-touch-icon.png assets autocomplete + boards ci dashboard deploy.html diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8a7cc690046..0f323a9e8b2 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -40,7 +40,7 @@ module Gitlab end def find_by_email - if auth_hash.has_email? + if auth_hash.has_attribute?(:email) user = ::User.find_by(email: auth_hash.email.downcase) user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user user diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb new file mode 100644 index 00000000000..d43eff5ba4a --- /dev/null +++ b/lib/gitlab/themes.rb @@ -0,0 +1,84 @@ +module Gitlab + # Module containing GitLab's application theme definitions and helper methods + # for accessing them. + module Themes + extend self + + # Theme ID used when no `default_theme` configuration setting is provided. + APPLICATION_DEFAULT = 1 + + # Struct class representing a single Theme + Theme = Struct.new(:id, :name, :css_class) + + # All available Themes + THEMES = [ + Theme.new(1, 'Indigo', 'ui_indigo'), + Theme.new(2, 'Dark', 'ui_dark'), + Theme.new(3, 'Light', 'ui_light'), + Theme.new(4, 'Blue', 'ui_blue'), + Theme.new(5, 'Green', 'ui_green') + ].freeze + + # Convenience method to get a space-separated String of all the theme + # classes that might be applied to the `body` element + # + # Returns a String + def body_classes + THEMES.collect(&:css_class).uniq.join(' ') + end + + # Get a Theme by its ID + # + # If the ID is invalid, returns the default Theme. + # + # id - Integer ID + # + # Returns a Theme + def by_id(id) + THEMES.detect { |t| t.id == id } || default + end + + # Returns the number of defined Themes + def count + THEMES.size + end + + # Get the default Theme + # + # Returns a Theme + def default + by_id(default_id) + end + + # Iterate through each Theme + # + # Yields the Theme object + def each(&block) + THEMES.each(&block) + end + + # Get the Theme for the specified user, or the default + # + # user - User record + # + # Returns a Theme + def for_user(user) + if user + by_id(user.theme_id) + else + default + end + end + + private + + def default_id + @default_id ||= begin + id = Gitlab.config.gitlab.default_theme.to_i + theme_ids = THEMES.map(&:id) + + theme_ids.include?(id) ? id : APPLICATION_DEFAULT + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index c81dc7e30d0..703adae12cb 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -9,7 +9,7 @@ module Gitlab end def self.valid?(url) - return false unless url + return false unless url.present? Addressable::URI.parse(url.strip) @@ -19,7 +19,12 @@ module Gitlab end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url.strip) + @url = Addressable::URI.parse(url.to_s.strip) + + %i[user password].each do |symbol| + credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) + end + @credentials = credentials end @@ -29,13 +34,13 @@ module Gitlab def masked_url url = @url.dup - url.password = "*****" unless url.password.nil? - url.user = "*****" unless url.user.nil? + url.password = "*****" if url.password.present? + url.user = "*****" if url.user.present? url.to_s end def credentials - @credentials ||= { user: @url.user, password: @url.password } + @credentials ||= { user: @url.user.presence, password: @url.password.presence } end def full_url @@ -47,8 +52,10 @@ module Gitlab def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url.user = credentials[:user] + @full_url end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 3cf26625108..36708078136 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -22,9 +22,13 @@ module Gitlab ci_builds: ::Ci::Build.count, ci_internal_pipelines: ::Ci::Pipeline.internal.count, ci_external_pipelines: ::Ci::Pipeline.external.count, + ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, + ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, ci_pipeline_schedules: ::Ci::PipelineSchedule.count, + auto_devops_enabled: ::ProjectAutoDevops.enabled.count, + auto_devops_disabled: ::ProjectAutoDevops.disabled.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e5ad9b5a40c..17550cf9074 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -124,7 +124,7 @@ module Gitlab def send_artifacts_entry(build, entry) params = { 'Archive' => build.artifacts_file.path, - 'Entry' => Base64.encode64(entry.path) + 'Entry' => Base64.encode64(entry.to_s) } [ diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb index 015c7ed1731..53a47eb0f42 100644 --- a/lib/system_check/app/init_script_up_to_date_check.rb +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -7,26 +7,22 @@ module SystemCheck set_skip_reason 'skipped (omnibus-gitlab has no init script)' def skip? - omnibus_gitlab? - end + return true if omnibus_gitlab? - def multi_check - recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + unless init_file_exists? + self.skip_reason = "can't check because of previous errors" - unless File.exist?(SCRIPT_PATH) - $stdout.puts "can't check because of previous errors".color(:magenta) - return + true end + end + + def check? + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') recipe_content = File.read(recipe_path) script_content = File.read(SCRIPT_PATH) - if recipe_content == script_content - $stdout.puts 'yes'.color(:green) - else - $stdout.puts 'no'.color(:red) - show_error - end + recipe_content == script_content end def show_error @@ -38,6 +34,12 @@ module SystemCheck ) fix_and_rerun end + + private + + def init_file_exists? + File.exist?(SCRIPT_PATH) + end end end end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 7f9e2ffffc2..0f5742dd67f 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -62,6 +62,25 @@ module SystemCheck call_or_return(@skip_reason) || 'skipped' end + # Define a reason why we skipped the SystemCheck (during runtime) + # + # This is used when you need dynamic evaluation like when you have + # multiple reasons why a check can fail + # + # @param [String] reason to be displayed + def skip_reason=(reason) + @skip_reason = reason + end + + # Skip reason defined during runtime + # + # This value have precedence over the one defined in the subclass + # + # @return [String] the reason + def skip_reason + @skip_reason + end + # Does the check support automatically repair routine? # # @return [Boolean] whether check implemented `#repair!` method or not diff --git a/lib/system_check/incoming_email/foreman_configured_check.rb b/lib/system_check/incoming_email/foreman_configured_check.rb new file mode 100644 index 00000000000..1db7bf2b782 --- /dev/null +++ b/lib/system_check/incoming_email/foreman_configured_check.rb @@ -0,0 +1,23 @@ +module SystemCheck + module IncomingEmail + class ForemanConfiguredCheck < SystemCheck::BaseCheck + set_name 'Foreman configured correctly?' + + def check? + path = Rails.root.join('Procfile') + + File.exist?(path) && File.read(path) =~ /^mail_room:/ + end + + def show_error + try_fixing_it( + 'Enable mail_room in your Procfile.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb new file mode 100644 index 00000000000..dee108d987b --- /dev/null +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -0,0 +1,45 @@ +module SystemCheck + module IncomingEmail + class ImapAuthenticationCheck < SystemCheck::BaseCheck + set_name 'IMAP server credentials are correct?' + + def check? + if mailbox_config + begin + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) + connected = true + rescue + connected = false + end + end + + connected + end + + def show_error + try_fixing_it( + 'Check that the information in config/gitlab.yml is correct' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mailbox_config + return @config if @config + + config_path = Rails.root.join('config', 'mail_room.yml').to_s + erb = ERB.new(File.read(config_path)) + erb.filename = config_path + config_file = YAML.load(erb.result) + + @config = config_file[:mailboxes]&.first + end + end + end +end diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb new file mode 100644 index 00000000000..ea23b8ef49c --- /dev/null +++ b/lib/system_check/incoming_email/initd_configured_check.rb @@ -0,0 +1,32 @@ +module SystemCheck + module IncomingEmail + class InitdConfiguredCheck < SystemCheck::BaseCheck + set_name 'Init.d configured correctly?' + + def skip? + omnibus_gitlab? + end + + def check? + mail_room_configured? + end + + def show_error + try_fixing_it( + 'Enable mail_room in the init.d configuration.' + ) + for_more_information( + 'doc/administration/reply_by_email.md' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + end + end +end diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb new file mode 100644 index 00000000000..c1807501829 --- /dev/null +++ b/lib/system_check/incoming_email/mail_room_running_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module IncomingEmail + class MailRoomRunningCheck < SystemCheck::BaseCheck + set_name 'MailRoom running?' + + def skip? + return true if omnibus_gitlab? + + unless mail_room_configured? + self.skip_reason = "can't check because of previous errors" + true + end + end + + def check? + mail_room_running? + end + + def show_error + try_fixing_it( + sudo_gitlab('RAILS_ENV=production bin/mail_room start') + ) + for_more_information( + see_installation_guide_section('Install Init Script'), + 'see log/mail_room.log for possible errors' + ) + fix_and_rerun + end + + private + + def mail_room_configured? + path = '/etc/default/gitlab' + File.exist?(path) && File.read(path).include?('mail_room_enabled=true') + end + + def mail_room_running? + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.include?("mail_room") + end + end + end +end diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb new file mode 100644 index 00000000000..b8446300f72 --- /dev/null +++ b/lib/system_check/orphans/namespace_check.rb @@ -0,0 +1,54 @@ +module SystemCheck + module Orphans + class NamespaceCheck < SystemCheck::BaseCheck + set_name 'Orphaned namespaces:' + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end + + clear_namespaces! # releases memory when check finishes + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned namespaces for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_namespaces(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + result << namespace + end + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def existing_namespaces + @namespaces ||= Namespace.where(parent: nil).all.pluck(:path) + end + + def clear_namespaces! + @namespaces = nil + end + end + end +end diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb new file mode 100644 index 00000000000..9b6b2429783 --- /dev/null +++ b/lib/system_check/orphans/repository_check.rb @@ -0,0 +1,68 @@ +module SystemCheck + module Orphans + class RepositoryCheck < SystemCheck::BaseCheck + set_name 'Orphaned repositories:' + attr_accessor :orphans + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + + repositories = disk_repositories(repository_storage['path']) + orphans = (repositories - fetch_repositories(storage_name)) + + print_orphans(orphans, storage_name) + end + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned repositories for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_repositories(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + fetch_disk_repositories(namespace_path).each do |repo| + result << "#{namespace}/#{File.basename(repo)}" + end + end + end + + def fetch_repositories(storage_name) + sql = " + SELECT + CONCAT(n.path, '/', p.path, '.git') repo, + CONCAT(n.path, '/', p.path, '.wiki.git') wiki + FROM projects p + JOIN namespaces n + ON (p.namespace_id = n.id AND + n.parent_id IS NULL) + WHERE (p.repository_storage LIKE ?) + " + + query = ActiveRecord::Base.send(:sanitize_sql_array, [sql, storage_name]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.connection.select_all(query).rows.try(:flatten!) || [] + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def fetch_disk_repositories(namespace_path) + Dir.glob(File.join(namespace_path, '*')) + end + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 6604b1078cf..00221f77cf4 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -23,7 +23,7 @@ module SystemCheck # # @param [BaseCheck] check class def <<(check) - raise ArgumentError unless check < BaseCheck + raise ArgumentError unless check.is_a?(Class) && check < BaseCheck @checks << check end @@ -48,7 +48,7 @@ module SystemCheck # When implements skip method, we run it first, and if true, skip the check if check.can_skip? && check.skip? - $stdout.puts check_klass.skip_reason.color(:magenta) + $stdout.puts check.skip_reason.try(:color, :magenta) || check_klass.skip_reason.color(:magenta) return end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 92a3f503fcb..dfade1f3885 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -309,133 +309,24 @@ namespace :gitlab do desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab - start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_imap_authentication + checks = [ + SystemCheck::IncomingEmail::ImapAuthenticationCheck + ] if Rails.env.production? - check_initd_configured_correctly - check_mail_room_running + checks << SystemCheck::IncomingEmail::InitdConfiguredCheck + checks << SystemCheck::IncomingEmail::MailRoomRunningCheck else - check_foreman_configured_correctly + checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck end - else - puts 'Reply by email is disabled in config/gitlab.yml' - end - - finished_checking "Reply by email" - end - - # Checks - ######################## - - def check_initd_configured_correctly - return if omnibus_gitlab? - - print "Init.d configured correctly? ... " - - path = "/etc/default/gitlab" - - if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in the init.d configuration." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - def check_foreman_configured_correctly - print "Foreman configured correctly? ... " - - path = Rails.root.join("Procfile") - - if File.exist?(path) && File.read(path) =~ /^mail_room:/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Enable mail_room in your Procfile." - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun - end - end - - def check_mail_room_running - return if omnibus_gitlab? - - print "MailRoom running? ... " - - path = "/etc/default/gitlab" - - unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") - puts "can't check because of previous errors".color(:magenta) - return - end - - if mail_room_running? - puts "yes".color(:green) + SystemCheck.run('Reply by email', checks) else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("RAILS_ENV=production bin/mail_room start") - ) - for_more_information( - see_installation_guide_section("Install Init Script"), - "see log/mail_room.log for possible errors" - ) - fix_and_rerun - end - end - - def check_imap_authentication - print "IMAP server credentials are correct? ... " - - config_path = Rails.root.join('config', 'mail_room.yml').to_s - erb = ERB.new(File.read(config_path)) - erb.filename = config_path - config_file = YAML.load(erb.result) - - config = config_file[:mailboxes].first - - if config - begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.starttls if config[:start_tls] - imap.login(config[:email], config[:password]) - connected = true - rescue - connected = false - end - end - - if connected - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Check that the information in config/gitlab.yml is correct" - ) - for_more_information( - "doc/administration/reply_by_email.md" - ) - fix_and_rerun + puts 'Reply by email is disabled in config/gitlab.yml' end end - - def mail_room_running? - ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.include?("mail_room") - end end namespace :ldap do @@ -507,6 +398,35 @@ namespace :gitlab do end end + namespace :orphans do + desc 'Gitlab | Check for orphaned namespaces and repositories' + task check: :environment do + warn_user_is_not_gitlab + checks = [ + SystemCheck::Orphans::NamespaceCheck, + SystemCheck::Orphans::RepositoryCheck + ] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned namespaces in the repositories path' + task check_namespaces: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::NamespaceCheck] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned repositories in the repositories path' + task check_repositories: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::RepositoryCheck] + + SystemCheck.run('Orphans', checks) + end + end + namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index dd1825c8a9e..44074397c05 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -9,5 +9,16 @@ namespace :gitlab do task data: :environment do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end + + desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' + task bump_test_version: :environment do + Dir.mktmpdir do |tmp_dir| + system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + end + + puts "Updated to #{Gitlab::ImportExport.version}" + end end end |