summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/api.rb1
-rw-r--r--lib/api/builds.rb46
-rw-r--r--lib/api/entities.rb23
-rw-r--r--lib/api/repositories.rb7
-rw-r--r--lib/api/runners.rb175
-rw-r--r--lib/banzai/filter/emoji_filter.rb3
-rw-r--r--lib/ci/api/builds.rb6
-rw-r--r--lib/ci/status.rb4
-rw-r--r--lib/gitlab/ldap/user.rb4
-rw-r--r--lib/gitlab/o_auth/user.rb13
-rw-r--r--lib/gitlab/push_data_builder.rb2
-rw-r--r--lib/gitlab/saml/user.rb47
-rw-r--r--lib/gitlab/workhorse.rb27
-rw-r--r--lib/tasks/cache.rake18
-rw-r--r--lib/tasks/gitlab/check.rake1
15 files changed, 351 insertions, 26 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 7efe0a0262f..7d65145176b 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -56,5 +56,6 @@ module API
mount Triggers
mount Builds
mount Variables
+ mount Runners
end
end
diff --git a/lib/api/builds.rb b/lib/api/builds.rb
index a8bd3842ce4..2b104f90aa7 100644
--- a/lib/api/builds.rb
+++ b/lib/api/builds.rb
@@ -60,6 +60,30 @@ module API
user_can_download_artifacts: can?(current_user, :read_build, user_project)
end
+ # Download the artifacts file from build
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Example Request:
+ # GET /projects/:id/builds/:build_id/artifacts
+ get ':id/builds/:build_id/artifacts' do
+ authorize_read_builds!
+
+ build = get_build(params[:build_id])
+ return not_found!(build) unless build
+
+ artifacts_file = build.artifacts_file
+
+ unless artifacts_file.file_storage?
+ return redirect_to build.artifacts_file.url
+ end
+
+ return not_found! unless artifacts_file.exists?
+
+ present_file!(artifacts_file.path, artifacts_file.filename)
+ end
+
# Get a trace of a specific build of a project
#
# Parameters:
@@ -115,13 +139,33 @@ module API
authorize_update_builds!
build = get_build(params[:build_id])
- return forbidden!('Build is not retryable') unless build && build.retryable?
+ return not_found!(build) unless build
+ return forbidden!('Build is not retryable') unless build.retryable?
build = Ci::Build.retry(build)
present build, with: Entities::Build,
user_can_download_artifacts: can?(current_user, :read_build, user_project)
end
+
+ # Erase build (remove artifacts and build trace)
+ #
+ # Parameters:
+ # id (required) - the id of a project
+ # build_id (required) - the id of a build
+ # example Request:
+ # post /projects/:id/build/:build_id/erase
+ post ':id/builds/:build_id/erase' do
+ authorize_update_builds!
+
+ build = get_build(params[:build_id])
+ return not_found!(build) unless build
+ return forbidden!('Build is not erasable!') unless build.erasable?
+
+ build.erase(erased_by: current_user)
+ present build, with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
end
helpers do
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index a9c09ffdb31..a3b5f1eb8d3 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -49,7 +49,7 @@ module API
expose :enable_ssl_verification
end
- class ForkedFromProject < Grape::Entity
+ class BasicProjectDetails < Grape::Entity
expose :id
expose :name, :name_with_namespace
expose :path, :path_with_namespace
@@ -67,7 +67,7 @@ module API
expose :shared_runners_enabled
expose :creator_id
expose :namespace
- expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ |project, options| project.forked? }
+ expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
expose :avatar_url
expose :star_count, :forks_count
expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
@@ -377,6 +377,24 @@ module API
expose :name
end
+ class RunnerDetails < Runner
+ expose :tag_list
+ expose :version, :revision, :platform, :architecture
+ expose :contacted_at
+ expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? }
+ expose :projects, with: Entities::BasicProjectDetails do |runner, options|
+ if options[:current_user].is_admin?
+ runner.projects
+ else
+ options[:current_user].authorized_projects.where(id: runner.projects)
+ end
+ end
+ end
+
+ class BuildArtifactFile < Grape::Entity
+ expose :filename, :size
+ end
+
class Build < Grape::Entity
expose :id, :status, :stage, :name, :ref, :tag, :coverage
expose :created_at, :started_at, :finished_at
@@ -388,6 +406,7 @@ module API
repo_obj.artifacts_download_url
end
end
+ expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? }
expose :commit, with: RepoCommit do |repo_obj, _options|
if repo_obj.respond_to?(:commit)
repo_obj.commit.commit_data
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index c95d2d2001d..0d0f0d4616d 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -98,11 +98,8 @@ module API
authorize! :download_code, user_project
begin
- ArchiveRepositoryService.new(
- user_project,
- params[:sha],
- params[:format]
- ).execute
+ RepositoryArchiveCacheWorker.perform_async
+ header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format])
rescue
not_found!('File')
end
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
new file mode 100644
index 00000000000..8ec91485b26
--- /dev/null
+++ b/lib/api/runners.rb
@@ -0,0 +1,175 @@
+module API
+ # Runners API
+ class Runners < Grape::API
+ before { authenticate! }
+
+ resource :runners do
+ # Get runners available for user
+ #
+ # Example Request:
+ # GET /runners
+ get do
+ runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared'])
+ present paginate(runners), with: Entities::Runner
+ end
+
+ # Get all runners - shared and specific
+ #
+ # Example Request:
+ # GET /runners/all
+ get 'all' do
+ authenticated_as_admin!
+ runners = filter_runners(Ci::Runner.all, params[:scope])
+ present paginate(runners), with: Entities::Runner
+ end
+
+ # Get runner's details
+ #
+ # Parameters:
+ # id (required) - The ID of ther runner
+ # Example Request:
+ # GET /runners/:id
+ get ':id' do
+ runner = get_runner(params[:id])
+ authenticate_show_runner!(runner)
+
+ present runner, with: Entities::RunnerDetails, current_user: current_user
+ end
+
+ # Update runner's details
+ #
+ # Parameters:
+ # id (required) - The ID of ther runner
+ # description (optional) - Runner's description
+ # active (optional) - Runner's status
+ # tag_list (optional) - Array of tags for runner
+ # Example Request:
+ # PUT /runners/:id
+ put ':id' do
+ runner = get_runner(params[:id])
+ authenticate_update_runner!(runner)
+
+ attrs = attributes_for_keys [:description, :active, :tag_list]
+ if runner.update(attrs)
+ present runner, with: Entities::RunnerDetails, current_user: current_user
+ else
+ render_validation_error!(runner)
+ end
+ end
+
+ # Remove runner
+ #
+ # Parameters:
+ # id (required) - The ID of ther runner
+ # Example Request:
+ # DELETE /runners/:id
+ delete ':id' do
+ runner = get_runner(params[:id])
+ authenticate_delete_runner!(runner)
+ runner.destroy!
+
+ present runner, with: Entities::Runner
+ end
+ end
+
+ resource :projects do
+ before { authorize_admin_project }
+
+ # Get runners available for project
+ #
+ # Example Request:
+ # GET /projects/:id/runners
+ get ':id/runners' do
+ runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope])
+ present paginate(runners), with: Entities::Runner
+ end
+
+ # Enable runner for project
+ #
+ # Parameters:
+ # id (required) - The ID of the project
+ # runner_id (required) - The ID of the runner
+ # Example Request:
+ # POST /projects/:id/runners/:runner_id
+ post ':id/runners' do
+ required_attributes! [:runner_id]
+
+ runner = get_runner(params[:runner_id])
+ authenticate_enable_runner!(runner)
+ Ci::RunnerProject.create(runner: runner, project: user_project)
+
+ present runner, with: Entities::Runner
+ end
+
+ # Disable project's runner
+ #
+ # Parameters:
+ # id (required) - The ID of the project
+ # runner_id (required) - The ID of the runner
+ # Example Request:
+ # DELETE /projects/:id/runners/:runner_id
+ delete ':id/runners/:runner_id' do
+ runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id])
+ not_found!('Runner') unless runner_project
+
+ runner = runner_project.runner
+ forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1
+
+ runner_project.destroy
+
+ present runner, with: Entities::Runner
+ end
+ end
+
+ helpers do
+ def filter_runners(runners, scope, options = {})
+ return runners unless scope.present?
+
+ available_scopes = ::Ci::Runner::AVAILABLE_SCOPES
+ if options[:without]
+ available_scopes = available_scopes - options[:without]
+ end
+
+ if (available_scopes & [scope]).empty?
+ render_api_error!('Scope contains invalid value', 400)
+ end
+
+ runners.send(scope)
+ end
+
+ def get_runner(id)
+ runner = Ci::Runner.find(id)
+ not_found!('Runner') unless runner
+ runner
+ end
+
+ def authenticate_show_runner!(runner)
+ return if runner.is_shared || current_user.is_admin?
+ forbidden!("No access granted") unless user_can_access_runner?(runner)
+ end
+
+ def authenticate_update_runner!(runner)
+ return if current_user.is_admin?
+ forbidden!("Runner is shared") if runner.is_shared?
+ forbidden!("No access granted") unless user_can_access_runner?(runner)
+ end
+
+ def authenticate_delete_runner!(runner)
+ return if current_user.is_admin?
+ forbidden!("Runner is shared") if runner.is_shared?
+ forbidden!("Runner associated with more than one project") if runner.projects.count > 1
+ forbidden!("No access granted") unless user_can_access_runner?(runner)
+ end
+
+ def authenticate_enable_runner!(runner)
+ forbidden!("Runner is shared") if runner.is_shared?
+ return if current_user.is_admin?
+ forbidden!("No access granted") unless user_can_access_runner?(runner)
+ end
+
+ def user_can_access_runner?(runner)
+ current_user.ci_authorized_runners.exists?(runner.id)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index 5952a031626..207437ba7cf 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -45,7 +45,8 @@ module Banzai
private
def emoji_url(name)
- emoji_path = "emoji/#{emoji_filename(name)}"
+ emoji_path = emoji_filename(name)
+
if context[:asset_host]
# Asset host is specified.
url_to_image(emoji_path)
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 416b0b5f0b4..2e9a5d311f9 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -38,6 +38,8 @@ module Ci
authenticate_runner!
update_runner_last_contact
build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
+ forbidden!('Build has been erased!') if build.erased?
+
build.update_attributes(trace: params[:trace]) if params[:trace]
case params[:state].to_s
@@ -99,6 +101,7 @@ module Ci
not_found! unless build
authenticate_build_token!(build)
forbidden!('Build is not running!') unless build.running?
+ forbidden!('Build has been erased!') if build.erased?
artifacts_upload_path = ArtifactUploader.artifacts_upload_path
artifacts = uploaded_file(:file, artifacts_upload_path)
@@ -143,7 +146,7 @@ module Ci
present_file!(artifacts_file.path, artifacts_file.filename)
end
- # Remove the artifacts file from build
+ # Remove the artifacts file from build - Runners only
#
# Parameters:
# id (required) - The ID of a build
@@ -156,6 +159,7 @@ module Ci
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
+
build.remove_artifacts_file!
build.remove_artifacts_metadata!
end
diff --git a/lib/ci/status.rb b/lib/ci/status.rb
index c02b3b8f3e4..3fb1fe29494 100644
--- a/lib/ci/status.rb
+++ b/lib/ci/status.rb
@@ -1,11 +1,9 @@
module Ci
class Status
def self.get_status(statuses)
- statuses.reject! { |status| status.try(&:allow_failure?) }
-
if statuses.none?
'skipped'
- elsif statuses.all?(&:success?)
+ elsif statuses.all? { |status| status.success? || status.ignored? }
'success'
elsif statuses.all?(&:pending?)
'pending'
diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb
index e044f0ecc6d..b84c81f1a6c 100644
--- a/lib/gitlab/ldap/user.rb
+++ b/lib/gitlab/ldap/user.rb
@@ -24,6 +24,10 @@ module Gitlab
update_user_attributes
end
+ def save
+ super('LDAP')
+ end
+
# instance methods
def gl_user
@gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index d87a72f7ba3..832fb08a526 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -26,7 +26,7 @@ module Gitlab
gl_user.try(:valid?)
end
- def save
+ def save(provider = 'OAuth')
unauthorized_to_create unless gl_user
if needs_blocking?
@@ -36,10 +36,10 @@ module Gitlab
gl_user.save!
end
- log.info "(OAuth) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}"
+ log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}"
gl_user
rescue ActiveRecord::RecordInvalid => e
- log.info "(OAuth) Error saving user: #{gl_user.errors.full_messages}"
+ log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}"
return self, e.record.errors
end
@@ -105,7 +105,12 @@ module Gitlab
end
def signup_enabled?
- Gitlab.config.omniauth.allow_single_sign_on
+ providers = Gitlab.config.omniauth.allow_single_sign_on
+ if providers.is_a?(Array)
+ providers.include?(auth_hash.provider)
+ else
+ providers
+ end
end
def block_after_signup?
diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index 1dad621aa00..da1c15fef61 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -22,6 +22,8 @@ module Gitlab
# }
#
def build(project, user, oldrev, newrev, ref, commits = [], message = nil)
+ commits = Array(commits)
+
# Total commits count
commits_count = commits.size
diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb
new file mode 100644
index 00000000000..b1e30110ef5
--- /dev/null
+++ b/lib/gitlab/saml/user.rb
@@ -0,0 +1,47 @@
+# SAML extension for User model
+#
+# * Find GitLab user based on SAML uid and provider
+# * Create new user from SAML data
+#
+module Gitlab
+ module Saml
+ class User < Gitlab::OAuth::User
+
+ def save
+ super('SAML')
+ end
+
+ def gl_user
+ @user ||= find_by_uid_and_provider
+
+ if auto_link_ldap_user?
+ @user ||= find_or_create_ldap_user
+ end
+
+ if auto_link_saml_enabled?
+ @user ||= find_by_email
+ end
+
+ if signup_enabled?
+ @user ||= build_new_user
+ end
+
+ @user
+ end
+
+ def find_by_email
+ if auth_hash.has_email?
+ user = ::User.find_by(email: auth_hash.email.downcase)
+ user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user
+ user
+ end
+ end
+
+ protected
+
+ def auto_link_saml_enabled?
+ Gitlab.config.omniauth.auto_link_saml_user
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index a23120a4176..c3ddd4c2680 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -3,19 +3,38 @@ require 'json'
module Gitlab
class Workhorse
+ SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
+
class << self
def send_git_blob(repository, blob)
- params_hash = {
+ params = {
'RepoPath' => repository.path_to_repo,
'BlobId' => blob.id,
}
- params = Base64.urlsafe_encode64(JSON.dump(params_hash))
[
- 'Gitlab-Workhorse-Send-Data',
- "git-blob:#{params}",
+ SEND_DATA_HEADER,
+ "git-blob:#{encode(params)}",
]
end
+
+ def send_git_archive(project, ref, format)
+ format ||= 'tar.gz'
+ format.downcase!
+ params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
+ raise "Repository or ref not found" if params.empty?
+
+ [
+ SEND_DATA_HEADER,
+ "git-archive:#{encode(params)}",
+ ]
+ end
+
+ protected
+
+ def encode(hash)
+ Base64.urlsafe_encode64(JSON.dump(hash))
+ end
end
end
end
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 1728dda72cf..9e2fb429d57 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -1,11 +1,21 @@
namespace :cache do
+ CLEAR_BATCH_SIZE = 1000 # The more the faster, but having too many can crash Ruby
+ REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
+
desc "GitLab | Clear redis cache"
task :clear => :environment do
- # Hack into Rails.cache until https://github.com/redis-store/redis-store/pull/225
- # is accepted (I hope) and we can update the redis-store gem.
redis_store = Rails.cache.instance_variable_get(:@data)
- redis_store.keys.each_slice(1000) do |key_slice|
- redis_store.del(*key_slice)
+ cursor = REDIS_SCAN_START_STOP
+ loop do
+ cursor, keys = redis_store.scan(
+ cursor,
+ match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*",
+ count: CLEAR_BATCH_SIZE
+ )
+
+ redis_store.del(*keys) if keys.any?
+
+ break if cursor == REDIS_SCAN_START_STOP
end
end
end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 54d95cd62a5..81099cb8ba9 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -16,7 +16,6 @@ namespace :gitlab do
check_git_config
check_database_config_exists
- check_database_is_not_sqlite
check_migrations_are_up
check_orphaned_group_members
check_gitlab_config_exists
View Lorry Controller Status