diff options
Diffstat (limited to 'lib')
34 files changed, 604 insertions, 127 deletions
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index e5793fbc5cb..710deba5ae3 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -20,6 +20,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 754c3d85a04..9117704aa46 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -14,13 +14,23 @@ module API end params do use :pagination - optional :scope, type: String, values: %w(running branches tags), - desc: 'Either running, branches, or tags' + optional :scope, type: String, values: %w[running pending finished branches tags], + desc: 'The scope of pipelines' + optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, + desc: 'The status of pipelines' + optional :ref, type: String, desc: 'The ref of pipelines' + optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' + optional :name, type: String, desc: 'The name of the user who triggered pipelines' + optional :username, type: String, desc: 'The username of the user who triggered pipelines' + optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', + desc: 'Order pipelines' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Sort pipelines' end get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index db4b31b55bc..9a6cb43abf7 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -26,6 +26,10 @@ module API params :optional_params do use :optional_params_ce end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end end resource :projects do @@ -56,10 +60,6 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' @@ -85,6 +85,7 @@ module API end params do use :collection_params + use :statistics_params end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails @@ -151,10 +152,13 @@ module API desc 'Get a single project' do success Entities::ProjectWithAccess end + params do + use :statistics_params + end get ":id" do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end desc 'Get events for a single project' do @@ -381,7 +385,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 3077240e650..1616142a619 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -23,6 +23,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index 82827249244..c48cbd2b765 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -21,7 +21,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute present paginate(pipelines), with: ::API::Entities::Pipeline end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index ba9748ada59..06cc704afc6 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -452,7 +452,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 84a28b33d7c..8b0662749fd 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -33,7 +33,8 @@ module Banzai { namespace: :owner }, { group: [:owners, :group_members] }, :invited_groups, - :project_members + :project_members, + :project_feature ] }), self.class.data_attribute diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index bae4db1ca4d..1501f64d537 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,16 +2,8 @@ class GroupUrlConstrainer def matches?(request) id = request.params[:id] - return false unless valid?(id) + return false unless DynamicPathValidator.valid?(id) Group.find_by_full_path(id).present? end - - private - - def valid?(id) - id.split('/').all? do |namespace| - NamespaceValidator.valid?(namespace) - end - end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index a10b4657d7d..d0ce2caffff 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,9 +4,7 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = namespace_path + '/' + project_path - unless ProjectPathValidator.valid?(project_path) - return false - end + return false unless DynamicPathValidator.valid?(full_path) Project.find_by_full_path(full_path).present? end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 6f799c2f031..2e073334abc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -37,6 +37,12 @@ module Gitlab !directory? end + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + def has_parent? nodes > 0 end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index a3cc350ef22..dad8c3cdf5b 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,7 +6,7 @@ module Gitlab def initialize(cron, cron_timezone = 'UTC') @cron = cron - @cron_timezone = cron_timezone + @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name end def next_time_from(time) @@ -24,8 +24,23 @@ module Gitlab private + # NOTE: + # cron_timezone can only accept timezones listed in TZInfo::Timezone. + # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, + # because Rufus::Scheduler only supports TZInfo::Timezone. + # + # For example, those codes have the same effect. + # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) + # Time.zone = 'America/Los_Angeles' (TZInfo::Timezone) + # + # However, try_parse_cron only accepts the latter format. + # try_parse_cron('* * * * *', 'Pacific Time (US & Canada)') -> Doesn't work + # try_parse_cron('* * * * *', 'America/Los_Angeles') -> Works + # If you want to know more, please take a look + # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb def try_parse_cron(cron, cron_timezone) - Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) rescue # noop end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6dabbe0264c..298b1a1f4e6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -498,6 +498,29 @@ module Gitlab columns(table).find { |column| column.name == name } end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..4fdcb682c2f --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,76 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..de4e6e7c404 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,131 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.map { |path| "%#{path}" } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%")) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..b9f4f3cff3c --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,72 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 32cece8316b..83440ae227d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -21,7 +21,7 @@ module Gitlab content_type: attachment.content_type } - link = ::Projects::UploadService.new(project, file).execute + link = UploadService.new(project, file).execute attachments << link if link ensure tmp.close! diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index c270c0ea9ff..0d6b08b5d29 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -57,9 +57,8 @@ module Gitlab end def key_from_additional_headers(mail) - references = ensure_references_array(mail.references) - - find_key_from_references(references) + find_key_from_references(mail) || + find_key_from_delivered_to_header(mail) end def ensure_references_array(references) @@ -75,12 +74,19 @@ module Gitlab end end - def find_key_from_references(references) - references.find do |mail_id| + def find_key_from_references(mail) + ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end + + def find_key_from_delivered_to_header(mail) + Array(mail[:delivered_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index a16d9fc2265..e3e36b35ce9 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -54,7 +54,7 @@ module Gitlab unicode_version: emoji_unicode_version(emoji_name) } - ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], data: data) + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data) end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index f6e4f279c06..aac210f19e8 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,31 +2,39 @@ module Gitlab module EtagCaching class Router Route = Struct.new(:regexp, :name) - - RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/rendered_title` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues rendered_title + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/commit/\S+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), 'commit_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/new\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), 'new_merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/\d+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), 'project_pipelines' ) ].freeze diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e8bb9e1f805..12458f9f410 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -128,6 +128,10 @@ module Gitlab encode! @name end + def truncated? + size && (size > loaded_size) + end + # Valid LFS object pointer is a text file consisting of # version # oid @@ -155,10 +159,14 @@ module Gitlab nil end - def truncated? - size && (size > loaded_size) + def external_storage + return unless lfs_pointer? + + :lfs end + alias_method :external_size, :lfs_size + private def has_lfs_version_key? diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 18eda0279f7..6a0f12b7e50 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,17 +45,13 @@ module Gitlab # Default branch in the repository def root_ref - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled| - # if is_enabled - # gitaly_ref_client.default_branch_name - # else - @root_ref ||= discover_default_branch - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| + if is_enabled + gitaly_ref_client.default_branch_name + else + discover_default_branch + end + end end # Alias to old method for compatibility @@ -72,17 +68,13 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - # Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # if is_enabled - # gitaly_ref_client.branch_names - # else - branches.map(&:name) - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names + else + branches.map(&:name) + end + end end # Returns an Array of Branches @@ -122,30 +114,43 @@ module Gitlab # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end # Returns an Array of tag names def tag_names - # Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # if is_enabled - # gitaly_ref_client.tag_names - # else - rugged.tags.map { |t| t.name } - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names + else + rugged.tags.map { |t| t.name } + end + end end # Returns an Array of Tags @@ -494,8 +499,9 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :none (default), :date, or :topo - # commit ordering types are documented here: + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) @@ -871,27 +877,6 @@ module Gitlab rugged.remotes[remote_name].push(refspecs) end - # Merge the +source_name+ branch into the +target_name+ branch. This is - # equivalent to `git merge --no_ff +source_name+`, since a merge commit - # is always created. - def merge(source_name, target_name, options = {}) - our_commit = rugged.branches[target_name].target - their_commit = rugged.branches[source_name].target - - raise "Invalid merge target" if our_commit.nil? - raise "Invalid merge source" if their_commit.nil? - - merge_index = rugged.merge_commits(our_commit, their_commit) - return false if merge_index.conflicts? - - actual_options = options.merge( - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - update_ref: "refs/heads/#{target_name}" - ) - Rugged::Commit.create(rugged, actual_options) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -1277,16 +1262,26 @@ module Gitlab @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) end - # Returns the `Rugged` sorting type constant for a given - # sort type key. Valid keys are `:none`, `:topo`, and `:date` - def rugged_sort_type(key) + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) @rugged_sort_types ||= { none: Rugged::SORT_NONE, topo: Rugged::SORT_TOPO, - date: Rugged::SORT_DATE + date: Rugged::SORT_DATE | Rugged::SORT_TOPO } - @rugged_sort_types.fetch(key, Rugged::SORT_NONE) + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) end end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index d3c0743db4e..f6c77ef1a3e 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -11,7 +11,9 @@ module Gitlab def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '') + branch_name = stub.find_default_branch_name(request).name + + Gitlab::Git.branch_name(branch_name) end def branch_names @@ -34,6 +36,14 @@ module Gitlab stub.find_ref_name(request).name end + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + private def consume_refs_response(response, prefix:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 899a6567768..3aac731e844 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -41,7 +41,6 @@ project_tree: - :statuses - triggers: - :trigger_schedule - - :deploy_keys - :services - :hooks - protected_branches: @@ -53,10 +52,6 @@ project_tree: # Only include the following attributes for the models specified. included_attributes: - project: - - :description - - :visibility_level - - :archived user: - :id - :email @@ -66,6 +61,30 @@ included_attributes: # Do not include the following attributes for the models specified. excluded_attributes: + project: + - :name + - :path + - :namespace_id + - :creator_id + - :import_url + - :import_status + - :avatar + - :import_type + - :import_source + - :import_error + - :mirror + - :runners_token + - :repository_storage + - :repository_read_only + - :lfs_enabled + - :import_jid + - :created_at + - :updated_at + - :import_jid + - :import_jid + - :id + - :star_count + - :last_activity_at snippets: - :expired_at merge_request_diff: @@ -94,3 +113,5 @@ methods: - :utf8_st_diffs merge_requests: - :diff_head_sha + project: + - :description_html
\ No newline at end of file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 2e349b5f9a9..84ab1977dfa 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -71,14 +71,14 @@ module Gitlab def restore_project return @project unless @tree_hash - @project.update(project_params) + @project.update_columns(project_params) @project end def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations - value.is_a?(Array) || key == key.singularize + value.respond_to?(:each) && !Project.column_names.include?(key) end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index a1e7159fe42..eb7f5120592 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -15,7 +15,10 @@ module Gitlab # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + attributes = @attributes_finder.find(:project) + project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} + + project_attributes.merge(include: build_hash(@tree)) rescue => e @shared.error(e) false diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 08b061d5e31..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 5293f5af12d..b5572a39d30 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -19,6 +19,7 @@ namespace :gemojione do entry = { category: emoji_hash['category'], moji: emoji_hash['moji'], + description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), digest: hash_digest, } diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 5476438b8fa..139ab70e125 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -65,6 +65,7 @@ namespace :gitlab do migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines .map { |file| Rails.root.join(file.strip).to_s } .select { |file| File.file?(file) } + .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 046780481ba..3c5bc0146a1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,18 +1,18 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| require 'toml' warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version - repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 95687066819..ee2cdcdea1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -41,8 +41,14 @@ namespace :gitlab do # Generate config.yml based on existing gitlab settings File.open("config.yml", "w+") {|f| f.puts config.to_yaml} - # Launch installation process - system(*%w(bin/install) + repository_storage_paths_args) + [ + %w(bin/install) + repository_storage_paths_args, + %w(bin/compile) + ].each do |cmd| + unless Kernel.system(*cmd) + raise "command failed: #{cmd.join(' ')}" + end + end end # (Re)create hooks diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index a00b02188cf..e7ac0b5859f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,16 +1,16 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version - repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 6ded519aff2..761f275d42a 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,7 +1,9 @@ require Rails.root.join('db/migrate/limits_to_mysql') +require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do puts "Adding limits to schema.rb for mysql" LimitsToMysql.new.up + MarkdownCacheLimitsToMysql.new.up end |