diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-10 15:07:47 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-10 15:07:47 +0000 |
commit | 8b1228b0d409d7751f01d9fb72ebfbbf62399486 (patch) | |
tree | 1b4126fe48d7666a90c0d7ee26230cf8379b6410 /lib | |
parent | 96b0c1245c93585a8b0fe23e22306d32ff4e4905 (diff) | |
download | gitlab-ce-8b1228b0d409d7751f01d9fb72ebfbbf62399486.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers.rb | 1 | ||||
-rw-r--r-- | lib/api/helpers/pagination.rb | 25 | ||||
-rw-r--r-- | lib/api/helpers/pagination_strategies.rb | 33 | ||||
-rw-r--r-- | lib/api/projects.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/job.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/release.rb | 46 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/release/assets.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/release/assets/link.rb | 32 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/release/assets/links.rb | 31 | ||||
-rw-r--r-- | lib/gitlab/ci/yaml_processor.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/config/entry/attributable.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/config/entry/configurable.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/git/rugged_impl/use_rugged.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/pagination/base.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/pagination/keyset.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/pagination/keyset/page.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/pagination/keyset/pager.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/pagination/keyset/request_context.rb | 2 |
18 files changed, 237 insertions, 73 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index bb61b4948b9..1fe2988ec1c 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -4,6 +4,7 @@ module API module Helpers include Gitlab::Utils include Helpers::Pagination + include Helpers::PaginationStrategies SUDO_HEADER = "HTTP_SUDO" GITLAB_SHARED_SECRET_HEADER = "Gitlab-Shared-Secret" diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 1b63e450a12..a6ae9a87f98 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,34 +3,9 @@ module API module Helpers module Pagination - # This returns an ActiveRecord relation def paginate(relation) Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) end - - # This applies pagination and executes the query - # It always returns an array instead of an ActiveRecord relation - def paginate_and_retrieve!(relation) - offset_or_keyset_pagination(relation).to_a - end - - private - - def offset_or_keyset_pagination(relation) - return paginate(relation) unless keyset_pagination_enabled? - - request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) - - unless Gitlab::Pagination::Keyset.available?(request_context, relation) - return error!('Keyset pagination is not yet available for this type of request', 405) - end - - Gitlab::Pagination::Keyset.paginate(request_context, relation) - end - - def keyset_pagination_enabled? - params[:pagination] == 'keyset' && Feature.enabled?(:api_keyset_pagination, default_enabled: true) - end end end end diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb new file mode 100644 index 00000000000..5f63635297a --- /dev/null +++ b/lib/api/helpers/pagination_strategies.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module API + module Helpers + module PaginationStrategies + def paginate_with_strategies(relation) + paginator = paginator(relation) + + yield(paginator.paginate(relation)).tap do |records, _| + paginator.finalize(records) + end + end + + def paginator(relation) + return Gitlab::Pagination::OffsetPagination.new(self) unless keyset_pagination_enabled? + + request_context = Gitlab::Pagination::Keyset::RequestContext.new(self) + + unless Gitlab::Pagination::Keyset.available?(request_context, relation) + return error!('Keyset pagination is not yet available for this type of request', 405) + end + + Gitlab::Pagination::Keyset::Pager.new(request_context) + end + + private + + def keyset_pagination_enabled? + params[:pagination] == 'keyset' && Feature.enabled?(:api_keyset_pagination, default_enabled: true) + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 68f199cc160..3e61b3c7f3b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -90,18 +90,22 @@ module API def present_projects(projects, options = {}) projects = reorder_projects(projects) projects = apply_filters(projects) - projects = paginate(projects) - projects, options = with_custom_attributes(projects, options) - options = options.reverse_merge( - with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, - statistics: params[:statistics], - current_user: current_user, - license: false - ) - options[:with] = Entities::BasicProjectDetails if params[:simple] + records, options = paginate_with_strategies(projects) do |projects| + projects, options = with_custom_attributes(projects, options) + + options = options.reverse_merge( + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user, + license: false + ) + options[:with] = Entities::BasicProjectDetails if params[:simple] + + [options[:with].prepare_relation(projects, options), options] + end - present options[:with].prepare_relation(projects, options), options + present records, options end def translate_params_for_compatibility(params) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 40918a6c85b..1b9daa2dbc7 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -17,7 +17,7 @@ module Gitlab allow_failure type stage when start_in artifacts cache dependencies before_script needs after_script variables environment coverage retry parallel extends interruptible timeout - resource_group].freeze + resource_group release].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -151,14 +151,18 @@ module Gitlab description: 'Coverage configuration for this job.', inherit: false + entry :release, Entry::Release, + description: 'This job will produce a release.', + inherit: false + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, :rules, - :parallel, :needs, :interruptible + :parallel, :needs, :interruptible, :release attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :extends, :start_in, :rules, - :interruptible, :timeout, :resource_group + :interruptible, :timeout, :resource_group, :release def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -243,6 +247,7 @@ module Gitlab interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, + release: release_value, after_script: after_script_value, ignore: ignored?, needs: needs_defined? ? needs_value : nil, diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb new file mode 100644 index 00000000000..3eceaa0ccd9 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a release configuration. + # + class Release < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[tag_name name description assets].freeze + attributes %i[tag_name name assets].freeze + + # Attributable description conflicts with + # ::Gitlab::Config::Entry::Node.description + def has_description? + true + end + + def description + config[:description] + end + + entry :assets, Entry::Release::Assets, description: 'Release assets.' + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :tag_name, presence: true + validates :description, type: String, presence: true + end + + helpers :assets + + def value + @config[:assets] = assets_value if @config.key?(:assets) + @config + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets.rb b/lib/gitlab/ci/config/entry/release/assets.rb new file mode 100644 index 00000000000..82ed39f51e0 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release assets. + # + class Release + class Assets < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[links].freeze + attributes ALLOWED_KEYS + + entry :links, Entry::Release::Assets::Links, description: 'Release assets:links.' + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :links, array_of_hashes: true, presence: true + end + + helpers :links + + def value + @config[:links] = links_value if @config.key?(:links) + @config + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets/link.rb b/lib/gitlab/ci/config/entry/release/assets/link.rb new file mode 100644 index 00000000000..8e8fcde16a3 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets/link.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release:assets:links. + # + class Release + class Assets + class Link < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[name url].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :url, presence: true, addressable_url: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/release/assets/links.rb b/lib/gitlab/ci/config/entry/release/assets/links.rb new file mode 100644 index 00000000000..b791d173d54 --- /dev/null +++ b/lib/gitlab/ci/config/entry/release/assets/links.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of release:assets:links. + # + class Release + class Assets + class Links < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + + entry :link, Entry::Release::Assets::Link, description: 'Release assets:links:link.' + + validations do + validates :config, type: Array, presence: true + end + + def skip_config_hash_validation? + true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 24e5f5fa9ab..080a8ac107d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -81,10 +81,15 @@ module Gitlab instance: job[:instance], start_in: job[:start_in], trigger: job[:trigger], - bridge_needs: job.dig(:needs, :bridge)&.first + bridge_needs: job.dig(:needs, :bridge)&.first, + release: release(job) }.compact }.compact end + def release(job) + job[:release] if Feature.enabled?(:ci_release_generation, default_enabled: false) + end + def stage_builds_attributes(stage) @jobs.values .select { |job| job[:stage] == stage } @@ -133,7 +138,6 @@ module Gitlab @jobs.each do |name, job| # logical validation for job - validate_job_stage!(name, job) validate_job_dependencies!(name, job) validate_job_needs!(name, job) diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 87bd257f69a..4deb233d10e 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -10,7 +10,7 @@ module Gitlab def attributes(*attributes) attributes.flatten.each do |attribute| if method_defined?(attribute) - raise ArgumentError, 'Method already defined!' + raise ArgumentError, "Method already defined: #{attribute}" end define_method(attribute) do diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index d5a093a469a..e7d441bb21c 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -5,7 +5,7 @@ module Gitlab module Entry ## # This mixin is responsible for adding DSL, which purpose is to - # simplifly process of adding child nodes. + # simplify the process of adding child nodes. # # This can be used only if parent node is a configuration entry that # holds a hash as a configuration value, for example: diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index ca5d533bd75..068aaf03c51 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -8,9 +8,17 @@ module Gitlab feature = Feature.get(feature_key) return feature.enabled? if Feature.persisted?(feature) + # Disable Rugged auto-detect(can_use_disk?) when Puma threads>1 + # https://gitlab.com/gitlab-org/gitlab/issues/119326 + return false if running_puma_with_multiple_threads? + Gitlab::GitalyClient.can_use_disk?(repo.storage) end + def running_puma_with_multiple_threads? + Gitlab::Runtime.puma? && ::Puma.cli_config.options[:max_threads] > 1 + end + def execute_rugged_call(method_name, *args) Gitlab::GitalyClient::StorageSettings.allow_disk_access do start = Gitlab::Metrics::System.monotonic_time diff --git a/lib/gitlab/pagination/base.rb b/lib/gitlab/pagination/base.rb index 90fa1f8d1ec..6402b9c0f76 100644 --- a/lib/gitlab/pagination/base.rb +++ b/lib/gitlab/pagination/base.rb @@ -3,6 +3,14 @@ module Gitlab module Pagination class Base + def paginate(relation) + raise NotImplementedError + end + + def finalize(records) + # Optional: Called with the actual set of records + end + private def per_page diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb index 5bd45fa9b56..8692f30e165 100644 --- a/lib/gitlab/pagination/keyset.rb +++ b/lib/gitlab/pagination/keyset.rb @@ -3,10 +3,6 @@ module Gitlab module Pagination module Keyset - def self.paginate(request_context, relation) - Gitlab::Pagination::Keyset::Pager.new(request_context).paginate(relation) - end - def self.available?(request_context, relation) order_by = request_context.page.order_by diff --git a/lib/gitlab/pagination/keyset/page.rb b/lib/gitlab/pagination/keyset/page.rb index 735f54faf0f..8070512f973 100644 --- a/lib/gitlab/pagination/keyset/page.rb +++ b/lib/gitlab/pagination/keyset/page.rb @@ -11,14 +11,13 @@ module Gitlab # Maximum number of records for a page MAXIMUM_PAGE_SIZE = 100 - attr_accessor :lower_bounds, :end_reached + attr_accessor :lower_bounds attr_reader :order_by - def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE, end_reached: false) + def initialize(order_by: {}, lower_bounds: nil, per_page: DEFAULT_PAGE_SIZE) @order_by = order_by.symbolize_keys @lower_bounds = lower_bounds&.symbolize_keys @per_page = per_page - @end_reached = end_reached end # Number of records to return per page @@ -28,17 +27,11 @@ module Gitlab [@per_page, MAXIMUM_PAGE_SIZE].min end - # Determine whether this page indicates the end of the collection - def end_reached? - @end_reached - end - # Construct a Page for the next page # Uses identical order_by/per_page information for the next page - def next(lower_bounds, end_reached) + def next(lower_bounds) dup.tap do |next_page| next_page.lower_bounds = lower_bounds&.symbolize_keys - next_page.end_reached = end_reached end end end diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb index 99b125cc2a0..6eaa7f3ba87 100644 --- a/lib/gitlab/pagination/keyset/pager.rb +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -14,27 +14,20 @@ module Gitlab # Validate assumption: The last two columns must match the page order_by validate_order!(relation) - # This performs the database query and retrieves records - # We retrieve one record more to check if we have data beyond this page - all_records = relation.limit(page.per_page + 1).to_a # rubocop: disable CodeReuse/ActiveRecord - - records_for_page = all_records.first(page.per_page) - - # If we retrieved more records than belong on this page, - # we know there's a next page - there_is_more = all_records.size > records_for_page.size - apply_headers(records_for_page.last, there_is_more) + relation.limit(page.per_page) # rubocop: disable CodeReuse/ActiveRecord + end - records_for_page + def finalize(records) + apply_headers(records.last) end private - def apply_headers(last_record_in_page, there_is_more) - end_reached = last_record_in_page.nil? || !there_is_more - lower_bounds = last_record_in_page&.slice(page.order_by.keys) + def apply_headers(last_record_in_page) + return unless last_record_in_page - next_page = page.next(lower_bounds, end_reached) + lower_bounds = last_record_in_page&.slice(page.order_by.keys) + next_page = page.next(lower_bounds) request.apply_headers(next_page) end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb index aeaed7587b3..8c8138b3076 100644 --- a/lib/gitlab/pagination/keyset/request_context.rb +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -68,8 +68,6 @@ module Gitlab end def pagination_links(next_page) - return if next_page.end_reached? - %(<#{page_href(next_page)}>; rel="next") end |