diff options
Diffstat (limited to 'lib')
26 files changed, 197 insertions, 61 deletions
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b888ede6fe8..8a54f7f3f05 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -47,6 +47,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/deploy_keys" do params[:key].strip! diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 8f3799417e3..605769eddde 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -142,7 +142,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 523f38d129e..56c597dffcb 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -138,8 +138,11 @@ module API return unless Gitlab::GitalyClient.enabled? + relative_path = Gitlab::RepoPath.strip_storage_path(params[:repo_path]) + project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, '')) + begin - Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive + Gitlab::GitalyClient::Notifications.new(project.repository_storage, relative_path).post_receive rescue GRPC::Unavailable => e render_api_error(e, 500) end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 4dce5dd130a..09053e615cb 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -26,6 +26,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' optional :iids, type: Array[Integer], desc: 'The IID array of issues' + optional :search, type: String, desc: 'Search issues for text present in the title or description' use :pagination end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0fbe1669d45..766fbea53e6 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -84,7 +84,7 @@ module API end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] end desc 'Create new project' do diff --git a/lib/api/users.rb b/lib/api/users.rb index 530ca0b5235..992a751b37d 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -341,7 +341,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index c5b37622d79..9b27411ae21 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -151,7 +151,7 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group).execute(current_user) + projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute projects = filter_projects(projects) entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index b753dbab381..ba9748ada59 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -107,7 +107,7 @@ module API end get '/visible' do entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity + present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity end desc 'Get a projects list for authenticated user' do diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb index 5e18cecc431..f4cda3b2eba 100644 --- a/lib/api/v3/users.rb +++ b/lib/api/v3/users.rb @@ -138,7 +138,7 @@ module API not_found!('User') unless user events = user.events. - merge(ProjectsFinder.new.execute(current_user)). + merge(ProjectsFinder.new(current_user: current_user).execute). references(:project). with_associations. recent diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ff580ec68f8..ee73fa91589 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -14,7 +14,7 @@ module Banzai def self.renderer @renderer ||= begin - renderer = Redcarpet::Render::HTML.new + renderer = Banzai::Renderer::HTML.new Redcarpet::Markdown.new(renderer, redcarpet_options) end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d5f9e252f62..522217deae4 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -24,10 +24,6 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class v-pre) - whitelist[:attributes]['span'] = %w(class) - # Allow table alignment whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -52,9 +48,6 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) - # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(self.class.clean_spans) - whitelist end @@ -84,21 +77,6 @@ module Banzai end end end - - def clean_spans - lambda do |env| - node = env[:node] - - return unless node.name == 'span' - return unless node.has_attribute?('class') - - unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } - node.remove_attribute('class') - end - - { node_whitelist: [node] } - end - end end end end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 9f09ca90697..7da565043d1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,7 +14,7 @@ module Banzai end def highlight_node(node) - language = node.attr('class') + language = node.attr('lang') code = node.text css_classes = "code highlight" lexer = lexer_for(language) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index fd4a6a107c2..bd4d1aa9ff8 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -9,9 +9,9 @@ module Banzai # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ - Filter::SyntaxHighlightFilter, Filter::PlantumlFilter, Filter::SanitizationFilter, + Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::UploadLinkFilter, diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb new file mode 100644 index 00000000000..252caa35947 --- /dev/null +++ b/lib/banzai/renderer/html.rb @@ -0,0 +1,13 @@ +module Banzai + module Renderer + class HTML < Redcarpet::Render::HTML + def block_code(code, lang) + lang_attr = lang ? %Q{ lang="#{lang}"} : '' + + "\n<pre>" \ + "<code#{lang_attr}>#{html_escape(code)}</code>" \ + "</pre>" + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fc445ab9483..525aa920328 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,6 +26,30 @@ module Gitlab add_index(table_name, column_name, options) end + # Removes an existed index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, :some_column + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index(table_name, column_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if Database.postgresql? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ column: column_name })) + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index ab8dfc67880..cd4e318033d 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -3,7 +3,8 @@ module Gitlab class Middleware RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) + %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z) ) def initialize(app) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index d3df3f1bca1..936606152e9 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -4,6 +4,8 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze + CommandError = Class.new(StandardError) + class << self def ref_name(ref) ref.sub(/\Arefs\/(tags|heads)\//, '') diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 32aebb6f6f0..2e4314932c8 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -25,9 +25,13 @@ module Gitlab # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(path) - @path = path - @name = path.split("/").last + def initialize(repository_storage, relative_path) + @repository_storage = repository_storage + @relative_path = relative_path + + storage_path = Gitlab.config.repositories.storages[@repository_storage]['path'] + @path = File.join(storage_path, @relative_path) + @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) end @@ -37,7 +41,15 @@ module Gitlab # Default branch in the repository def root_ref - @root_ref ||= discover_default_branch + @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled| + if is_enabled + gitaly_ref_client.default_branch_name + else + discover_default_branch + end + end + rescue GRPC::BadStatus => e + raise CommandError.new(e) end # Alias to old method for compatibility @@ -54,7 +66,15 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - branches.map(&:name) + Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names + else + branches.map(&:name) + end + end + rescue GRPC::BadStatus => e + raise CommandError.new(e) end # Returns an Array of Branches @@ -107,7 +127,15 @@ module Gitlab # Returns an Array of tag names def tag_names - rugged.tags.map { |t| t.name } + Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names + else + rugged.tags.map { |t| t.name } + end + end + rescue GRPC::BadStatus => e + raise CommandError.new(e) end # Returns an Array of Tags @@ -1202,6 +1230,10 @@ module Gitlab diff.find_similar!(break_rewrites: break_rewrites) diff.each_patch end + + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(@repository_storage, @relative_path) + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index fe15fb12adb..bcdf1b1faa8 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,11 +4,23 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - def self.configure_channel(storage, address) - @addresses ||= {} - @addresses[storage] = address - @channels ||= {} - @channels[storage] = new_channel(address) + # This function is not thread-safe because it updates Hashes in instance variables. + def self.configure_channels + @addresses = {} + @channels = {} + Gitlab.config.repositories.storages.each do |name, params| + address = params['gitaly_address'] + unless address.present? + raise "storage #{name.inspect} is missing a gitaly_address" + end + + unless URI(address).scheme.in?(%w(tcp unix)) + raise "Unsupported Gitaly address: #{address.inspect}" + end + + @addresses[name] = address + @channels[name] = new_channel(address) + end end def self.new_channel(address) @@ -21,10 +33,26 @@ module Gitlab end def self.get_channel(storage) + if !Rails.env.production? && @channels.nil? + # In development mode the Rails auto-loader may reset the instance + # variables of this class. What we do here is not thread-safe. In normal + # circumstances (including production) these instance variables have + # been initialized from config/initializers. + configure_channels + end + @channels[storage] end def self.get_address(storage) + if !Rails.env.production? && @addresses.nil? + # In development mode the Rails auto-loader may reset the instance + # variables of this class. What we do here is not thread-safe. In normal + # circumstances (including development) these instance variables have + # been initialized from config/initializers. + configure_channels + end + @addresses[storage] end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb index cbfb129c002..f0d93ded91b 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -3,18 +3,13 @@ module Gitlab class Notifications attr_accessor :stub - def initialize(repo_path) - full_path = Gitlab::RepoPath.strip_storage_path(repo_path). - sub(/\.git\z/, '').sub(/\.wiki\z/, '') - @project = Project.find_by_full_path(full_path) - - channel = GitalyClient.get_channel(@project.repository_storage) - @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel) + def initialize(repository_storage, relative_path) + @channel, @repository = Util.process_path(repository_storage, relative_path) + @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: @channel) end def post_receive - repository = Gitaly::Repository.new(path: @project.repository.path_to_repo) - request = Gitaly::PostReceiveRequest.new(repository: repository) + request = Gitaly::PostReceiveRequest.new(repository: @repository) @stub.post_receive(request) end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb new file mode 100644 index 00000000000..bfc5fa573c7 --- /dev/null +++ b/lib/gitlab/gitaly_client/ref.rb @@ -0,0 +1,35 @@ +module Gitlab + module GitalyClient + class Ref + attr_accessor :stub + + def initialize(repository_storage, relative_path) + @channel, @repository = Util.process_path(repository_storage, relative_path) + @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: @channel) + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @repository) + stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '') + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @repository) + consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/') + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @repository) + consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') + end + + private + + def consume_refs_response(response, prefix:) + response.flat_map do |r| + r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb new file mode 100644 index 00000000000..d272c25d1f9 --- /dev/null +++ b/lib/gitlab/gitaly_client/util.rb @@ -0,0 +1,13 @@ +module Gitlab + module GitalyClient + module Util + def self.process_path(repository_storage, relative_path) + channel = GitalyClient.get_channel(repository_storage) + storage_path = Gitlab.config.repositories.storages[repository_storage]['path'] + repository = Gitaly::Repository.new(path: File.join(storage_path, relative_path)) + + [channel, repository] + end + end + end +end diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index c44bb1cd14d..f0c50584f07 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -12,7 +12,7 @@ module Gitlab value = -1 end - response.headers[HEADER_NAME] = value + response.headers[HEADER_NAME] = value.to_s end def self.polling_enabled? diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 11e5f1b645c..ca8d3271541 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -72,6 +72,8 @@ module Gitlab # job_ids - The Sidekiq job IDs to check. # # Returns an array of true or false indicating job completion. + # true = job is still running + # false = job completed def self.job_status(job_ids) keys = job_ids.map { |jid| key_for(jid) } @@ -82,6 +84,17 @@ module Gitlab end end + # Returns the JIDs that are completed + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of completed JIDs + def self.completed_jids(job_ids) + Sidekiq.redis do |redis| + job_ids.reject { |jid| redis.exists(key_for(jid)) } + end + end + def self.key_for(jid) STATUS_KEY % jid end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index d47609f490d..00983b3284a 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -2,7 +2,9 @@ module Gitlab module SidekiqStatus class ClientMiddleware def call(_, job, _, _) - Gitlab::SidekiqStatus.set(job['jid']) + status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION + + Gitlab::SidekiqStatus.set(job['jid'], status_expiration) yield end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 08011301d3c..a8a7bf9bc12 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -45,12 +45,7 @@ module Gitlab raise "Unsupported action: #{action}" end - if feature_enabled - params[:GitalyAddress] = address - # TODO deprecate GitalySocketPath once GITLAB_WORKHORSE_VERSION points - # to a version that supports GitalyAddress. - params[:GitalySocketPath] = URI(address).path - end + params[:GitalyAddress] = address if feature_enabled end params |