diff options
Diffstat (limited to 'lib')
29 files changed, 426 insertions, 169 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 7b9de7c9598..87915b19480 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -5,15 +5,14 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project access requests viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # GET /groups/:id/access_requests - # GET /projects/:id/access_requests + desc "Gets a list of access requests for a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end get ":id/access_requests" do source = find_source(source_type, params[:id]) @@ -23,14 +22,10 @@ module API present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end - # Request access to the group/project - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # POST /groups/:id/access_requests - # POST /projects/:id/access_requests + desc "Requests access for the authenticated user to a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end post ":id/access_requests" do source = find_source(source_type, params[:id]) access_requester = source.request_access(current_user) @@ -42,42 +37,34 @@ module API end end - # Approve a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # access_level (optional) - Access level - # - # Example Request: - # PUT /groups/:id/access_requests/:user_id/approve - # PUT /projects/:id/access_requests/:user_id/approve + desc 'Approves an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + end put ':id/access_requests/:user_id/approve' do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, params).execute + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute status :created present member.user, with: Entities::Member, member: member end - # Deny a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # - # Example Request: - # DELETE /groups/:id/access_requests/:user_id - # DELETE /projects/:id/access_requests/:user_id + desc 'Denies an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + end delete ":id/access_requests/:user_id" do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - access_requester = source.requesters.find_by!(user_id: params[:user_id]) - - ::Members::DestroyService.new(access_requester, current_user).execute + ::Members::DestroyService.new(source, current_user, params). + execute(:requesters) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index cb47ec8f33f..0bbf73a1b63 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -43,6 +43,7 @@ module API mount ::API::Groups mount ::API::Internal mount ::API::Issues + mount ::API::Boards mount ::API::Keys mount ::API::Labels mount ::API::LicenseTemplates diff --git a/lib/api/boards.rb b/lib/api/boards.rb new file mode 100644 index 00000000000..4d5d144a02e --- /dev/null +++ b/lib/api/boards.rb @@ -0,0 +1,115 @@ +module API + # Boards API + class Boards < Grape::API + before { authenticate! } + + resource :projects do + # Get the project board + get ':id/boards' do + authorize!(:read_board, user_project) + present [user_project.board], with: Entities::Board + end + + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.board + if params[:board_id].to_i == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + # Get the lists of a project board + # Does not include `backlog` and `done` lists + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: Entities::List + end + + # Get a list of a project board + get '/lists/:list_id' do + authorize!(:read_board, user_project) + present board_lists.find(params[:list_id]), with: Entities::List + end + + # Create a new board list + # + # Parameters: + # id (required) - The ID of a project + # label_id (required) - The ID of an existing label + # Example Request: + # POST /projects/:id/boards/:board_id/lists + post '/lists' do + required_attributes! [:label_id] + + unless user_project.labels.exists?(params[:label_id]) + render_api_error!({ error: "Label not found!" }, 400) + end + + authorize!(:admin_list, user_project) + + list = ::Boards::Lists::CreateService.new(user_project, current_user, + { label_id: params[:label_id] }).execute + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + # Moves a board list to a new position + # + # Parameters: + # id (required) - The ID of a project + # board_id (required) - The ID of a board + # position (required) - The position of the list + # Example Request: + # PUT /projects/:id/boards/:board_id/lists/:list_id + put '/lists/:list_id' do + list = project_board.lists.movable.find(params[:list_id]) + + authorize!(:admin_list, user_project) + + moved = ::Boards::Lists::MoveService.new(user_project, current_user, + { position: params[:position].to_i }).execute(list) + + if moved + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + # Delete a board list + # + # Parameters: + # id (required) - The ID of a project + # board_id (required) - The ID of a board + # list_id (required) - The ID of a board list + # Example Request: + # DELETE /projects/:id/boards/:board_id/lists/:list_id + delete "/lists/:list_id" do + list = board_lists.find_by(id: params[:list_id]) + + authorize!(:admin_list, user_project) + + if list + destroyed_list = ::Boards::Lists::DestroyService.new( + user_project, current_user).execute(list) + present destroyed_list, with: Entities::List + else + not_found!('List') + end + end + end + end + end +end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index b4eaf1813d4..14ddc8c9a62 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -29,6 +29,42 @@ module API present commits, with: Entities::RepoCommit end + desc 'Commit multiple file changes as one commit' do + detail 'This feature was introduced in GitLab 8.13' + end + + params do + requires :id, type: Integer, desc: 'The project ID' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array, desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared(params) + attrs[:source_branch] = attrs[:branch_name] + attrs[:target_branch] = attrs[:branch_name] + attrs[:actions].map! do |action| + action[:action] = action[:action].to_sym + action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/') + action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/') + action + end + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + # Get a specific commit of a project # # Parameters: diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 04437322ec1..feaa0c213bf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -432,8 +432,11 @@ module API end end - class Label < Grape::Entity + class LabelBasic < Grape::Entity expose :name, :color, :description + end + + class Label < LabelBasic expose :open_issues_count, :closed_issues_count, :open_merge_requests_count expose :subscribed do |label, options| @@ -441,6 +444,19 @@ module API end end + class List < Grape::Entity + expose :id + expose :label, using: Entities::LabelBasic + expose :position + end + + class Board < Grape::Entity + expose :id + expose :lists, using: Entities::List do |board| + board.lists.destroyable + end + end + class Compare < Grape::Entity expose :commit, using: Entities::RepoCommit do |compare, options| Commit.decorate(compare.commits, nil).last diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 953fa474e88..bfb89475025 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -6,6 +6,8 @@ module API resource :groups do # Get a groups list # + # Parameters: + # skip_groups (optional) - Array of group ids to exclude from list # Example Request: # GET /groups get do @@ -16,6 +18,7 @@ module API end @groups = @groups.search(params[:search]) if params[:search].present? + @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? @groups = paginate @groups present @groups, with: Entities::Group end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 714d4ea3dc6..8b8c4eb4d46 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -21,8 +21,11 @@ module API end # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints def find_user_from_warden - warden ? warden.authenticate : nil + warden.try(:authenticate) if request.get? || request.head? end def find_user_by_private_token diff --git a/lib/api/members.rb b/lib/api/members.rb index a18ce769e29..b80818f0eb6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -5,16 +5,16 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project members viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # query - Query string - # - # Example Request: - # GET /groups/:id/members - # GET /projects/:id/members + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + end get ":id/members" do source = find_source(source_type, params[:id]) @@ -25,15 +25,12 @@ module API present users, with: Entities::Member, source: source end - # Get a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # GET /groups/:id/members/:user_id - # GET /projects/:id/members/:user_id + desc 'Gets a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -43,26 +40,25 @@ module API present member.user, with: Entities::Member, member: member end - # Add a new group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the new member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # POST /groups/:id/members - # POST /projects/:id/members + desc 'Adds a member to a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] member = source.members.find_by(user_id: params[:user_id]) - # This is to ensure back-compatibility but 409 behavior should be used - # for both project and group members in 9.0! + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! conflict!('Member already exists') if source_type == 'group' && member unless member @@ -79,21 +75,17 @@ module API end end - # Update a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # access_level (required) - A valid access level - # expires_at (optional) - Date string in the format YEAR-MONTH-DAY - # - # Example Request: - # PUT /groups/:id/members/:user_id - # PUT /projects/:id/members/:user_id + desc 'Updates a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end put ":id/members/:user_id" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] member = source.members.find_by!(user_id: params[:user_id]) attrs = attributes_for_keys [:access_level, :expires_at] @@ -108,18 +100,12 @@ module API end end - # Remove a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # DELETE /groups/:id/members/:user_id - # DELETE /projects/:id/members/:user_id + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - required_attributes! [:user_id] # This is to ensure back-compatibility but find_by! should be used # in that casse in 9.0! @@ -134,7 +120,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(member, current_user).execute + ::Members::DestroyService.new(source, current_user, declared(params)).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 50d3729449e..fe981d7b9fa 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -4,20 +4,18 @@ module API before { authenticate! } resource :namespaces do - # Get a namespaces list - # - # Example Request: - # GET /namespaces + desc 'Get a namespaces list' do + success Entities::Namespace + end + params do + optional :search, type: String, desc: "Search query for namespaces" + end get do - @namespaces = if current_user.admin - Namespace.all - else - current_user.namespaces - end - @namespaces = @namespaces.search(params[:search]) if params[:search].present? - @namespaces = paginate @namespaces + namespaces = current_user.admin ? Namespace.all : current_user.namespaces + + namespaces = namespaces.search(params[:search]) if params[:search].present? - present @namespaces, with: Entities::Namespace + present paginate(namespaces), with: Entities::Namespace end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 2470362e019..af1e575fc89 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,7 +25,7 @@ module Banzai return if customized?(whitelist[:transformers]) # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) + whitelist[:attributes]['pre'] = %w(class v-pre) whitelist[:attributes]['span'] = %w(class) # Allow table alignment diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fcdb496aed2..026b81ac175 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -30,7 +30,7 @@ module Banzai # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index e1ca7f4d24b..c6302b586d3 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -106,13 +106,17 @@ module Banzai project = context[:project] author = context[:author] - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) + if author && !project.team.member?(author) + link_text + else + url = urls.namespace_project_url(project.namespace, project, + only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + data = data_attribute(project: project.id, author: author.try(:id)) + text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, text, 'All Project and Group Members') + end end def link_to_namespace(namespace, link_text: nil) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 59f85416ee5..ed87a2603e8 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,10 +12,9 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact(save: false) - update_runner_info required_attributes! [:token] not_found! unless current_runner.active? + update_runner_info build = Ci::RegisterBuildService.new.execute(current_runner) @@ -41,10 +40,11 @@ module Ci # PUT /builds/:id put ":id" do authenticate_runner! - update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) forbidden!('Build has been erased!') if build.erased? + update_runner_info + build.update_attributes(trace: params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 23353c62885..e608f5f6cad 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 40 * 60 + UPDATE_RUNNER_EVERY = 10 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -30,14 +30,22 @@ module Ci token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact(save: true) - # Use a random threshold to prevent beating DB updates - # it generates a distribution between: [40m, 80m] + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.contacted_at = Time.now - current_runner.save if current_runner.changed? && save - end + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age end def build_not_found! @@ -57,11 +65,6 @@ module Ci attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) end - def update_runner_info - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - def max_artifacts_size current_application_settings.max_artifacts_size.megabytes.to_i end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index e33ac61f5ae..7f424b74efb 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -102,9 +102,19 @@ module Gitlab def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = api.send(method, *args, &block) - yield data + data = api.send(method, *args) + return data unless data.is_a?(Array) + if block_given? + yield data + each_response_page(&block) + else + each_response_page { |page| data.concat(page) } + data + end + end + + def each_response_page last_response = api.last_response while last_response.rels[:next] diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b8321244473..4b70f33a851 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -170,10 +170,9 @@ module Gitlab end def import_wiki - unless project.wiki_enabled? + unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 605abfabdab..a2410068845 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,7 +1,7 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data def initialize(repo, name, namespace, current_user, session_data) @repo = repo @@ -12,24 +12,37 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, - name: @name, - path: @name, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, + visibility_level: visibility_level, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: import_url, + skip_wiki: skip_wiki ).execute + end + + private - # If repo has wiki we'll import it later - if repo.has_wiki? && project - project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) - end + def import_url + repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + end - project + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? end end end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3e5d728f3bc..f8809db21aa 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,19 +5,61 @@ module Gitlab def identify(identifier, project, newrev) if identifier.blank? # Local push from gitlab - email = project.commit(newrev).author_email rescue nil - User.find_by(email: email) if email - + identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http - user_id = identifier.gsub("user-", "") - User.find_by(id: user_id) - + identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh - key_id = identifier.gsub("key-", "") - Key.find_by(id: key_id).try(:user) + identify_using_ssh_key(identifier) + end + end + + # Tries to identify a user based on a commit SHA. + def identify_using_commit(project, ref) + commit = project.commit(ref) + + return if !commit || !commit.author_email + + email = commit.author_email + + identify_with_cache(:email, email) do + User.find_by(email: email) end end + + # Tries to identify a user based on a user identifier (e.g. "user-123"). + def identify_using_user(identifier) + user_id = identifier.gsub("user-", "") + + identify_with_cache(:user, user_id) do + User.find_by(id: user_id) + end + end + + # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). + def identify_using_ssh_key(identifier) + key_id = identifier.gsub("key-", "") + + identify_with_cache(:ssh_key, key_id) do + User.find_by_ssh_key_id(key_id) + end + end + + def identify_with_cache(category, key) + if identification_cache[category].key?(key) + identification_cache[category][key] + else + identification_cache[category][key] = yield + end + end + + def identification_cache + @identification_cache ||= { + email: {}, + user: {}, + ssh_key: {} + } + end end end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb new file mode 100644 index 00000000000..b9e4042220a --- /dev/null +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -0,0 +1,13 @@ +module Gitlab + module ImportExport + class AttributeCleaner + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + + def self.clean!(relation_hash:) + relation_hash.reject! do |key, _value| + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + end + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e522a0fc8f6..f00c7460e82 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport module CommandLineUtil + DEFAULT_MODE = 0700 + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -21,6 +23,11 @@ module Gitlab execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) end + def mkdir_p(path) + FileUtils.mkdir_p(path, mode: DEFAULT_MODE) + FileUtils.chmod(DEFAULT_MODE, path) + end + private def tar_with_options(archive:, dir:, options:) @@ -45,7 +52,7 @@ module Gitlab # if we are copying files, create the destination folder destination_folder = File.file?(source) ? File.dirname(destination) : destination - FileUtils.mkdir_p(destination_folder) + mkdir_p(destination_folder) FileUtils.copy_entry(source, destination) true end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index eca6e5b6d51..113895ba22c 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -15,7 +15,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) wait_for_archived_file do decompress_archive diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 35ff134ea19..5a109f24f9f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -110,9 +110,10 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => restored_project.id), + relation_hash: relation_hash, members_mapper: members_mapper, - user: @user) + user: @user, + project_id: restored_project.id) end relation_hash_list.is_a?(Array) ? relation_array : relation_array.first diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 9153088e966..2fbf437ec26 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport class ProjectTreeSaver + include Gitlab::ImportExport::CommandLineUtil + attr_reader :full_path def initialize(project:, shared:) @@ -10,7 +12,7 @@ module Gitlab end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 354ccd64696..9300f789e1b 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -13,6 +13,8 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze + BUILD_MODELS = %w[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze @@ -25,9 +27,9 @@ module Gitlab new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id') + @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -153,7 +155,11 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= begin + Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash) + + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end end def set_st_diffs diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d1e33ea8678..48a9a6fa5e2 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -12,7 +12,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - FileUtils.mkdir_p(path_to_repo) + mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks rescue => e diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 331e14021e6..a7028a32570 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -20,7 +20,7 @@ module Gitlab private def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 9b642d740b7..7cf88298642 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,12 +1,14 @@ module Gitlab module ImportExport class VersionSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(shared:) @shared = shared end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 6107420e4dd..1e6722a7bba 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -9,7 +9,7 @@ module Gitlab end def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 3faab937726..c649da8c426 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -24,10 +24,20 @@ module Gitlab end def with - @pool ||= ConnectionPool.new { ::Redis.new(params) } + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @pool.with { |redis| yield redis } end + def pool_size + if Sidekiq.server? + # the pool will be used in a multi-threaded context + Sidekiq.options[:concurrency] + 5 + else + # probably this is a Unicorn process, so single threaded + 5 + end + end + def _raw_config return @_raw_config if defined?(@_raw_config) |