summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/entities.rb5
-rw-r--r--lib/api/helpers.rb18
-rw-r--r--lib/api/merge_requests.rb7
-rw-r--r--lib/api/repositories.rb10
-rw-r--r--lib/api/session.rb2
-rw-r--r--lib/backup/manager.rb21
-rw-r--r--lib/ci/api/entities.rb2
-rw-r--r--lib/ci/gitlab_ci_yaml_processor.rb26
-rw-r--r--lib/gitlab/auth.rb6
-rw-r--r--lib/gitlab/backend/grack_auth.rb2
-rw-r--r--lib/gitlab/database/migration_helpers.rb13
-rw-r--r--lib/gitlab/github_import/client.rb35
-rw-r--r--lib/gitlab/github_import/importer.rb62
-rw-r--r--lib/gitlab/sanitizers/svg.rb48
-rw-r--r--lib/gitlab/workhorse.rb18
15 files changed, 170 insertions, 105 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 50d69274b2e..14370ac218d 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -179,6 +179,11 @@ module API
expose :upvotes, :downvotes
end
+ class ExternalIssue < Grape::Entity
+ expose :title
+ expose :id
+ end
+
class MergeRequest < ProjectEntity
expose :target_branch, :source_branch
expose :upvotes, :downvotes
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index a179fe9f2f9..8c4a707e7ee 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -420,5 +420,23 @@ module API
error!(errors[:access_level], 422) if errors[:access_level].any?
not_found!(errors)
end
+
+ def send_git_blob(repository, blob)
+ env['api.format'] = :txt
+ content_type 'text/plain'
+ header(*Gitlab::Workhorse.send_git_blob(repository, blob))
+ end
+
+ def send_git_archive(repository, ref:, format:)
+ header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
+ end
+
+ def issue_entity(project)
+ if project.has_external_issue_tracker?
+ Entities::ExternalIssue
+ else
+ Entities::Issue
+ end
+ end
end
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 43221d5622a..0e94efd4acd 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -228,11 +228,10 @@ module API
# Merge request can not be merged
# because user dont have permissions to push into target branch
unauthorized! unless merge_request.can_be_merged_by?(current_user)
- not_allowed! if !merge_request.open? || merge_request.work_in_progress?
- merge_request.check_if_can_be_merged
+ not_allowed! unless merge_request.mergeable_state?
- render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged?
+ render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?
if params[:sha] && merge_request.source_sha != params[:sha]
render_api_error!("SHA does not match HEAD of source branch: #{merge_request.source_sha}", 409)
@@ -330,7 +329,7 @@ module API
get "#{path}/closes_issues" do
merge_request = user_project.merge_requests.find(params[:merge_request_id])
issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
- present paginate(issues), with: Entities::Issue, current_user: current_user
+ present paginate(issues), with: issue_entity(user_project), current_user: current_user
end
end
end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 9cb14e95ebc..f55aceed92c 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -56,8 +56,7 @@ module API
blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath])
not_found! "File" unless blob
- content_type 'text/plain'
- header(*Gitlab::Workhorse.send_git_blob(repo, blob))
+ send_git_blob repo, blob
end
# Get a raw blob contents by blob sha
@@ -80,10 +79,7 @@ module API
not_found! 'Blob' unless blob
- env['api.format'] = :txt
-
- content_type blob.mime_type
- header(*Gitlab::Workhorse.send_git_blob(repo, blob))
+ send_git_blob repo, blob
end
# Get a an archive of the repository
@@ -98,7 +94,7 @@ module API
authorize! :download_code, user_project
begin
- header(*Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]))
+ send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
rescue
not_found!('File')
end
diff --git a/lib/api/session.rb b/lib/api/session.rb
index 56e69b2366f..56c202f1294 100644
--- a/lib/api/session.rb
+++ b/lib/api/session.rb
@@ -11,7 +11,7 @@ module API
# Example Request:
# POST /session
post "/session" do
- user = Gitlab::Auth.find_in_gitlab_or_ldap(params[:email] || params[:login], params[:password])
+ user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password])
return unauthorized! unless user
present user, with: Entities::UserLogin
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 9dd665441a0..2ff3e3bdfb0 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -38,7 +38,6 @@ module Backup
end
def upload(tar_file)
- remote_directory = Gitlab.config.backup.upload.remote_directory
$progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
connection_settings = Gitlab.config.backup.upload.connection
@@ -47,8 +46,7 @@ module Backup
return
end
- connection = ::Fog::Storage.new(connection_settings)
- directory = connection.directories.create(key: remote_directory)
+ directory = connect_to_remote_directory(connection_settings)
if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
@@ -155,6 +153,23 @@ module Backup
private
+ def connect_to_remote_directory(connection_settings)
+ connection = ::Fog::Storage.new(connection_settings)
+
+ # We only attempt to create the directory for local backups. For AWS
+ # and other cloud providers, we cannot guarantee the user will have
+ # permission to create the bucket.
+ if connection.service == ::Fog::Storage::Local
+ connection.directories.create(key: remote_directory)
+ else
+ connection.directories.get(remote_directory)
+ end
+ end
+
+ def remote_directory
+ Gitlab.config.backup.upload.remote_directory
+ end
+
def backup_contents
folders_to_backup + archives_to_backup + ["backup_information.yml"]
end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b25e0e573a8..a902ced35d7 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -56,7 +56,7 @@ module Ci
class TriggerRequest < Grape::Entity
expose :id, :variables
- expose :commit, using: Commit
+ expose :pipeline, using: Commit, as: :commit
end
end
end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 130f5b0892e..40a5d180fd0 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -8,6 +8,8 @@ module Ci
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
:allow_failure, :type, :stage, :when, :artifacts, :cache,
:dependencies, :before_script, :after_script, :variables]
+ ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
+ ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when]
attr_reader :before_script, :after_script, :image, :services, :path, :cache
@@ -135,6 +137,12 @@ module Ci
end
def validate_global_cache!
+ @cache.keys.each do |key|
+ unless ALLOWED_CACHE_KEYS.include? key
+ raise ValidationError, "#{name} cache unknown parameter #{key}"
+ end
+ end
+
if @cache[:key] && !validate_string(@cache[:key])
raise ValidationError, "cache:key parameter should be a string"
end
@@ -200,7 +208,7 @@ module Ci
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
- if job[:when] && !job[:when].in?(%w(on_success on_failure always))
+ if job[:when] && !job[:when].in?(%w[on_success on_failure always])
raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always"
end
end
@@ -233,6 +241,12 @@ module Ci
end
def validate_job_cache!(name, job)
+ job[:cache].keys.each do |key|
+ unless ALLOWED_CACHE_KEYS.include? key
+ raise ValidationError, "#{name} job: cache unknown parameter #{key}"
+ end
+ end
+
if job[:cache][:key] && !validate_string(job[:cache][:key])
raise ValidationError, "#{name} job: cache:key parameter should be a string"
end
@@ -247,6 +261,12 @@ module Ci
end
def validate_job_artifacts!(name, job)
+ job[:artifacts].keys.each do |key|
+ unless ALLOWED_ARTIFACTS_KEYS.include? key
+ raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
+ end
+ end
+
if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
end
@@ -258,6 +278,10 @@ module Ci
if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
end
+
+ if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
+ raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
+ end
end
def validate_job_dependencies!(name, job)
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 076e2af7d38..db1704af75e 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -3,14 +3,14 @@ module Gitlab
Result = Struct.new(:user, :type)
class << self
- def find(login, password, project:, ip:)
+ def find_for_git_client(login, password, project:, ip:)
raise "Must provide an IP for rate limiting" if ip.nil?
result = Result.new
if valid_ci_request?(login, password, project)
result.type = :ci
- elsif result.user = find_in_gitlab_or_ldap(login, password)
+ elsif result.user = find_with_user_password(login, password)
result.type = :gitlab_or_ldap
elsif result.user = oauth_access_token_check(login, password)
result.type = :oauth
@@ -20,7 +20,7 @@ module Gitlab
result
end
- def find_in_gitlab_or_ldap(login, password)
+ def find_with_user_password(login, password)
user = User.by_login(login)
# If no user is found, or it's an LDAP server, try LDAP.
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 9e09d2e118d..adbf5941a96 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -95,7 +95,7 @@ module Grack
end
def authenticate_user(login, password)
- user = Gitlab::Auth.find_in_gitlab_or_ldap(login, password)
+ user = Gitlab::Auth.find_with_user_password(login, password)
unless user
user = oauth_access_token_check(login, password)
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 978c3f7896d..dd3ff0ab18b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -31,8 +31,6 @@ module Gitlab
# Any data inserted while running this method (or after it has finished
# running) is _not_ updated automatically.
#
- # This method _only_ updates rows where the column's value is set to NULL.
- #
# table - The name of the table.
# column - The name of the column to update.
# value - The value for the column.
@@ -55,10 +53,10 @@ module Gitlab
first['count'].
to_i
- # Update in batches of 5%
+ # Update in batches of 5% until we run out of any rows to update.
batch_size = ((total / 100.0) * 5.0).ceil
- while processed < total
+ loop do
start_row = exec_query(%Q{
SELECT id
FROM #{quoted_table}
@@ -66,6 +64,9 @@ module Gitlab
LIMIT 1 OFFSET #{processed}
}).to_hash.first
+ # There are no more rows to process
+ break unless start_row
+
stop_row = exec_query(%Q{
SELECT id
FROM #{quoted_table}
@@ -126,6 +127,8 @@ module Gitlab
begin
transaction do
update_column_in_batches(table, column, default)
+
+ change_column_null(table, column, false) unless allow_null
end
# We want to rescue _all_ exceptions here, even those that don't inherit
# from StandardError.
@@ -134,8 +137,6 @@ module Gitlab
raise error
end
-
- change_column_null(table, column, false) unless allow_null
end
end
end
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 67988ea3460..d325eca6d99 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -1,6 +1,9 @@
module Gitlab
module GithubImport
class Client
+ GITHUB_SAFE_REMAINING_REQUESTS = 100
+ GITHUB_SAFE_SLEEP_TIME = 500
+
attr_reader :client, :api
def initialize(access_token)
@@ -11,7 +14,7 @@ module Gitlab
)
if access_token
- ::Octokit.auto_paginate = true
+ ::Octokit.auto_paginate = false
@api = ::Octokit::Client.new(
access_token: access_token,
@@ -36,7 +39,7 @@ module Gitlab
def method_missing(method, *args, &block)
if api.respond_to?(method)
- api.send(method, *args, &block)
+ request { api.send(method, *args, &block) }
else
super(method, *args, &block)
end
@@ -55,6 +58,34 @@ module Gitlab
def github_options
config["args"]["client_options"].deep_symbolize_keys
end
+
+ def rate_limit
+ api.rate_limit!
+ end
+
+ def rate_limit_exceed?
+ rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
+ end
+
+ def rate_limit_sleep_time
+ rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
+ end
+
+ def request
+ sleep rate_limit_sleep_time if rate_limit_exceed?
+
+ data = yield
+
+ last_response = api.last_response
+
+ while last_response.rels[:next]
+ sleep rate_limit_sleep_time if rate_limit_exceed?
+ last_response = last_response.rels[:next].get
+ data.concat(last_response.data) if last_response.data.is_a?(Array)
+ end
+
+ data
+ end
end
end
end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 5ef9d66ba68..e5cf66a0371 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,9 +3,6 @@ module Gitlab
class Importer
include Gitlab::ShellAdapter
- GITHUB_SAFE_REMAINING_REQUESTS = 100
- GITHUB_SAFE_SLEEP_TIME = 500
-
attr_reader :client, :project, :repo, :repo_url
def initialize(project)
@@ -28,52 +25,12 @@ module Gitlab
private
- def turn_auto_pagination_off!
- client.auto_paginate = false
- end
-
- def turn_auto_pagination_on!
- client.auto_paginate = true
- end
-
- def rate_limit
- client.rate_limit!
- end
-
- def rate_limit_exceed?
- rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
- end
-
- def rate_limit_sleep_time
- rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
- end
-
- def paginate
- turn_auto_pagination_off!
-
- sleep rate_limit_sleep_time if rate_limit_exceed?
-
- data = yield
-
- last_response = client.last_response
-
- while last_response.rels[:next]
- sleep rate_limit_sleep_time if rate_limit_exceed?
- last_response = last_response.rels[:next].get
- data.concat(last_response.data) if last_response.data.is_a?(Array)
- end
-
- turn_auto_pagination_on!
-
- data
- end
-
def credentials
@credentials ||= project.import_data.credentials if project.import_data
end
def import_labels
- labels = paginate { client.labels(repo, per_page: 100) }
+ labels = client.labels(repo, per_page: 100)
labels.each { |raw| LabelFormatter.new(project, raw).create! }
true
@@ -82,7 +39,7 @@ module Gitlab
end
def import_milestones
- milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
+ milestones = client.milestones(repo, state: :all, per_page: 100)
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
true
@@ -91,9 +48,9 @@ module Gitlab
end
def import_issues
- data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
+ issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
- data.each do |raw|
+ issues.each do |raw|
gh_issue = IssueFormatter.new(project, raw)
if gh_issue.valid?
@@ -112,7 +69,7 @@ module Gitlab
hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
disable_webhooks(hooks)
- pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
+ pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
@@ -146,14 +103,12 @@ module Gitlab
def update_webhooks(hooks, options)
hooks.each do |hook|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.edit_hook(repo, hook.id, hook.name, hook.config, options)
end
end
def restore_branches(branches)
branches.each do |name, sha|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha)
end
@@ -162,15 +117,12 @@ module Gitlab
def clean_up_restored_branches(branches)
branches.each do |name, _|
- sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name)
end
end
def apply_labels(issuable)
- sleep rate_limit_sleep_time if rate_limit_exceed?
-
issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0
@@ -183,12 +135,12 @@ module Gitlab
end
def import_comments(issuable)
- comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
+ comments = client.issue_comments(repo, issuable.iid, per_page: 100)
create_comments(issuable, comments)
end
def import_comments_on_diff(merge_request)
- comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
+ comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
create_comments(merge_request, comments)
end
diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb
index 5e95f6c0529..8304b9a482c 100644
--- a/lib/gitlab/sanitizers/svg.rb
+++ b/lib/gitlab/sanitizers/svg.rb
@@ -12,23 +12,45 @@ module Gitlab
def scrub(node)
unless Whitelist::ALLOWED_ELEMENTS.include?(node.name)
node.unlink
- else
- node.attributes.each do |attr_name, attr|
- valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name]
-
- unless valid_attributes && valid_attributes.include?(attr_name)
- if Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name) &&
- attr_name.start_with?('data-')
- # Arbitrary data attributes are allowed. Verify that the attribute
- # is a valid data attribute.
- attr.unlink unless attr_name =~ DATA_ATTR_PATTERN
- else
- attr.unlink
- end
+ return
+ end
+
+ valid_attributes = Whitelist::ALLOWED_ATTRIBUTES[node.name]
+ return unless valid_attributes
+
+ node.attribute_nodes.each do |attr|
+ attr_name = attribute_name_with_namespace(attr)
+
+ if valid_attributes.include?(attr_name)
+ attr.unlink if unsafe_href?(attr)
+ else
+ # Arbitrary data attributes are allowed.
+ unless allows_data_attribute?(node) && data_attribute?(attr)
+ attr.unlink
end
end
end
end
+
+ def attribute_name_with_namespace(attr)
+ if attr.namespace
+ "#{attr.namespace.prefix}:#{attr.name}"
+ else
+ attr.name
+ end
+ end
+
+ def allows_data_attribute?(node)
+ Whitelist::ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS.include?(node.name)
+ end
+
+ def unsafe_href?(attr)
+ attribute_name_with_namespace(attr) == 'xlink:href' && !attr.value.start_with?('#')
+ end
+
+ def data_attribute?(attr)
+ attr.name.start_with?('data-') && attr.name =~ DATA_ATTR_PATTERN && attr.namespace.nil?
+ end
end
end
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 56af739b1ef..388f84dbe0e 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -21,27 +21,29 @@ module Gitlab
[
SEND_DATA_HEADER,
- "git-blob:#{encode(params)}",
+ "git-blob:#{encode(params)}"
]
end
- def send_git_archive(project, ref, format)
+ def send_git_archive(repository, ref:, format:)
format ||= 'tar.gz'
format.downcase!
- params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
+ params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
raise "Repository or ref not found" if params.empty?
[
SEND_DATA_HEADER,
- "git-archive:#{encode(params)}",
+ "git-archive:#{encode(params)}"
]
end
- def send_git_diff(repository, from, to)
+ def send_git_diff(repository, diff_refs)
+ from, to = diff_refs
+
params = {
- 'RepoPath' => repository.path_to_repo,
- 'ShaFrom' => from,
- 'ShaTo' => to
+ 'RepoPath' => repository.path_to_repo,
+ 'ShaFrom' => from.sha,
+ 'ShaTo' => to.sha
}
[
View Lorry Controller Status