summaryrefslogtreecommitdiff
path: root/lib/gitlab
diff options
context:
space:
mode:
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/backend/shell.rb46
-rw-r--r--lib/gitlab/github_import/client.rb14
-rw-r--r--lib/gitlab/github_import/importer.rb3
-rw-r--r--lib/gitlab/github_import/project_creator.rb35
-rw-r--r--lib/gitlab/identifier.rb58
-rw-r--r--lib/gitlab/import_export/attribute_cleaner.rb13
-rw-r--r--lib/gitlab/import_export/command_line_util.rb9
-rw-r--r--lib/gitlab/import_export/file_importer.rb2
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb5
-rw-r--r--lib/gitlab/import_export/project_tree_saver.rb4
-rw-r--r--lib/gitlab/import_export/relation_factory.rb12
-rw-r--r--lib/gitlab/import_export/repo_restorer.rb2
-rw-r--r--lib/gitlab/import_export/repo_saver.rb2
-rw-r--r--lib/gitlab/import_export/version_saver.rb4
-rw-r--r--lib/gitlab/import_export/wiki_repo_saver.rb2
-rw-r--r--lib/gitlab/redis.rb12
16 files changed, 172 insertions, 51 deletions
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 79eac66b364..d0060fbaca1 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -17,6 +17,18 @@ module Gitlab
end
class << self
+ def secret_token
+ @secret_token ||= begin
+ File.read(Gitlab.config.gitlab_shell.secret_file).chomp
+ end
+ end
+
+ def ensure_secret_token!
+ return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret'))
+
+ generate_and_link_secret_token
+ end
+
def version_required
@version_required ||= File.read(Rails.root.
join('GITLAB_SHELL_VERSION')).strip
@@ -25,6 +37,25 @@ module Gitlab
def strip_key(key)
key.split(/ /)[0, 2].join(' ')
end
+
+ private
+
+ # Create (if necessary) and link the secret token file
+ def generate_and_link_secret_token
+ secret_file = Gitlab.config.gitlab_shell.secret_file
+ shell_path = Gitlab.config.gitlab_shell.path
+
+ unless File.size?(secret_file)
+ # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
+ token = SecureRandom.hex(16)
+ File.write(secret_file, token)
+ end
+
+ link_path = File.join(shell_path, '.gitlab_shell_secret')
+ if File.exist?(shell_path) && !File.exist?(link_path)
+ FileUtils.symlink(secret_file, link_path)
+ end
+ end
end
# Init new repository
@@ -201,21 +232,6 @@ module Gitlab
File.exist?(full_path(storage, dir_name))
end
- # Create (if necessary) and link the secret token file
- def generate_and_link_secret_token
- secret_file = Gitlab.config.gitlab_shell.secret_file
- unless File.size?(secret_file)
- # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
- token = SecureRandom.hex(16)
- File.write(secret_file, token)
- end
-
- link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret')
- if File.exist?(gitlab_shell_path) && !File.exist?(link_path)
- FileUtils.symlink(secret_file, link_path)
- end
- end
-
protected
def gitlab_shell_path
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index e33ac61f5ae..7f424b74efb 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -102,9 +102,19 @@ module Gitlab
def request(method, *args, &block)
sleep rate_limit_sleep_time if rate_limit_exceed?
- data = api.send(method, *args, &block)
- yield data
+ data = api.send(method, *args)
+ return data unless data.is_a?(Array)
+ if block_given?
+ yield data
+ each_response_page(&block)
+ else
+ each_response_page { |page| data.concat(page) }
+ data
+ end
+ end
+
+ def each_response_page
last_response = api.last_response
while last_response.rels[:next]
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index b8321244473..4b70f33a851 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -170,10 +170,9 @@ module Gitlab
end
def import_wiki
- unless project.wiki_enabled?
+ unless project.wiki.repository_exists?
wiki = WikiFormatter.new(project)
gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url)
- project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED)
end
rescue Gitlab::Shell::Error => e
# GitHub error message when the wiki repo has not been created,
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index 605abfabdab..a2410068845 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,7 +1,7 @@
module Gitlab
module GithubImport
class ProjectCreator
- attr_reader :repo, :namespace, :current_user, :session_data
+ attr_reader :repo, :name, :namespace, :current_user, :session_data
def initialize(repo, name, namespace, current_user, session_data)
@repo = repo
@@ -12,24 +12,37 @@ module Gitlab
end
def execute
- project = ::Projects::CreateService.new(
+ ::Projects::CreateService.new(
current_user,
- name: @name,
- path: @name,
+ name: name,
+ path: name,
description: repo.description,
namespace_id: namespace.id,
- visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility,
+ visibility_level: visibility_level,
import_type: "github",
import_source: repo.full_name,
- import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
+ import_url: import_url,
+ skip_wiki: skip_wiki
).execute
+ end
+
+ private
- # If repo has wiki we'll import it later
- if repo.has_wiki? && project
- project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED)
- end
+ def import_url
+ repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+ end
+
+ def visibility_level
+ repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility
+ end
- project
+ #
+ # If the GitHub project repository has wiki, we should not create the
+ # default wiki. Otherwise the GitHub importer will fail because the wiki
+ # repository already exist.
+ #
+ def skip_wiki
+ repo.has_wiki?
end
end
end
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index 3e5d728f3bc..f8809db21aa 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -5,19 +5,61 @@ module Gitlab
def identify(identifier, project, newrev)
if identifier.blank?
# Local push from gitlab
- email = project.commit(newrev).author_email rescue nil
- User.find_by(email: email) if email
-
+ identify_using_commit(project, newrev)
elsif identifier =~ /\Auser-\d+\Z/
# git push over http
- user_id = identifier.gsub("user-", "")
- User.find_by(id: user_id)
-
+ identify_using_user(identifier)
elsif identifier =~ /\Akey-\d+\Z/
# git push over ssh
- key_id = identifier.gsub("key-", "")
- Key.find_by(id: key_id).try(:user)
+ identify_using_ssh_key(identifier)
+ end
+ end
+
+ # Tries to identify a user based on a commit SHA.
+ def identify_using_commit(project, ref)
+ commit = project.commit(ref)
+
+ return if !commit || !commit.author_email
+
+ email = commit.author_email
+
+ identify_with_cache(:email, email) do
+ User.find_by(email: email)
end
end
+
+ # Tries to identify a user based on a user identifier (e.g. "user-123").
+ def identify_using_user(identifier)
+ user_id = identifier.gsub("user-", "")
+
+ identify_with_cache(:user, user_id) do
+ User.find_by(id: user_id)
+ end
+ end
+
+ # Tries to identify a user based on an SSH key identifier (e.g. "key-123").
+ def identify_using_ssh_key(identifier)
+ key_id = identifier.gsub("key-", "")
+
+ identify_with_cache(:ssh_key, key_id) do
+ User.find_by_ssh_key_id(key_id)
+ end
+ end
+
+ def identify_with_cache(category, key)
+ if identification_cache[category].key?(key)
+ identification_cache[category][key]
+ else
+ identification_cache[category][key] = yield
+ end
+ end
+
+ def identification_cache
+ @identification_cache ||= {
+ email: {},
+ user: {},
+ ssh_key: {}
+ }
+ end
end
end
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
new file mode 100644
index 00000000000..b9e4042220a
--- /dev/null
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -0,0 +1,13 @@
+module Gitlab
+ module ImportExport
+ class AttributeCleaner
+ ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES
+
+ def self.clean!(relation_hash:)
+ relation_hash.reject! do |key, _value|
+ key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index e522a0fc8f6..f00c7460e82 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -1,6 +1,8 @@
module Gitlab
module ImportExport
module CommandLineUtil
+ DEFAULT_MODE = 0700
+
def tar_czf(archive:, dir:)
tar_with_options(archive: archive, dir: dir, options: 'czf')
end
@@ -21,6 +23,11 @@ module Gitlab
execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
end
+ def mkdir_p(path)
+ FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
+ FileUtils.chmod(DEFAULT_MODE, path)
+ end
+
private
def tar_with_options(archive:, dir:, options:)
@@ -45,7 +52,7 @@ module Gitlab
# if we are copying files, create the destination folder
destination_folder = File.file?(source) ? File.dirname(destination) : destination
- FileUtils.mkdir_p(destination_folder)
+ mkdir_p(destination_folder)
FileUtils.copy_entry(source, destination)
true
end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index eca6e5b6d51..113895ba22c 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -15,7 +15,7 @@ module Gitlab
end
def import
- FileUtils.mkdir_p(@shared.export_path)
+ mkdir_p(@shared.export_path)
wait_for_archived_file do
decompress_archive
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 35ff134ea19..5a109f24f9f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -110,9 +110,10 @@ module Gitlab
def create_relation(relation, relation_hash_list)
relation_array = [relation_hash_list].flatten.map do |relation_hash|
Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
- relation_hash: relation_hash.merge('project_id' => restored_project.id),
+ relation_hash: relation_hash,
members_mapper: members_mapper,
- user: @user)
+ user: @user,
+ project_id: restored_project.id)
end
relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 9153088e966..2fbf437ec26 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -1,6 +1,8 @@
module Gitlab
module ImportExport
class ProjectTreeSaver
+ include Gitlab::ImportExport::CommandLineUtil
+
attr_reader :full_path
def initialize(project:, shared:)
@@ -10,7 +12,7 @@ module Gitlab
end
def save
- FileUtils.mkdir_p(@shared.export_path)
+ mkdir_p(@shared.export_path)
File.write(full_path, project_json_tree)
true
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 354ccd64696..9300f789e1b 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -13,6 +13,8 @@ module Gitlab
USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
+ PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze
+
BUILD_MODELS = %w[Ci::Build commit_status].freeze
IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
@@ -25,9 +27,9 @@ module Gitlab
new(*args).create
end
- def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+ def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:)
@relation_name = OVERRIDES[relation_sym] || relation_sym
- @relation_hash = relation_hash.except('id', 'noteable_id')
+ @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id)
@members_mapper = members_mapper
@user = user
@imported_object_retries = 0
@@ -153,7 +155,11 @@ module Gitlab
end
def parsed_relation_hash
- @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+ @parsed_relation_hash ||= begin
+ Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash)
+
+ @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
+ end
end
def set_st_diffs
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index d1e33ea8678..48a9a6fa5e2 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -12,7 +12,7 @@ module Gitlab
def restore
return true unless File.exist?(@path_to_bundle)
- FileUtils.mkdir_p(path_to_repo)
+ mkdir_p(path_to_repo)
git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks
rescue => e
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 331e14021e6..a7028a32570 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -20,7 +20,7 @@ module Gitlab
private
def bundle_to_disk
- FileUtils.mkdir_p(@shared.export_path)
+ mkdir_p(@shared.export_path)
git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
rescue => e
@shared.error(e)
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
index 9b642d740b7..7cf88298642 100644
--- a/lib/gitlab/import_export/version_saver.rb
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -1,12 +1,14 @@
module Gitlab
module ImportExport
class VersionSaver
+ include Gitlab::ImportExport::CommandLineUtil
+
def initialize(shared:)
@shared = shared
end
def save
- FileUtils.mkdir_p(@shared.export_path)
+ mkdir_p(@shared.export_path)
File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
rescue => e
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 6107420e4dd..1e6722a7bba 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -9,7 +9,7 @@ module Gitlab
end
def bundle_to_disk(full_path)
- FileUtils.mkdir_p(@shared.export_path)
+ mkdir_p(@shared.export_path)
git_bundle(repo_path: path_to_repo, bundle_path: full_path)
rescue => e
@shared.error(e)
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 3faab937726..c649da8c426 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -24,10 +24,20 @@ module Gitlab
end
def with
- @pool ||= ConnectionPool.new { ::Redis.new(params) }
+ @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
@pool.with { |redis| yield redis }
end
+ def pool_size
+ if Sidekiq.server?
+ # the pool will be used in a multi-threaded context
+ Sidekiq.options[:concurrency] + 5
+ else
+ # probably this is a Unicorn process, so single threaded
+ 5
+ end
+ end
+
def _raw_config
return @_raw_config if defined?(@_raw_config)
View Lorry Controller Status