summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorTomasz Maczukin <tomasz@maczukin.pl>2015-11-20 00:29:04 +0100
committerTomasz Maczukin <tomasz@maczukin.pl>2015-11-20 00:29:04 +0100
commit85ad95be741848fbf15a01789f065e001326cefa (patch)
tree496395e235a41b51e69c47e73e3440e5e4105666 /lib
parent1144b70ab624ee1c1e7f2de0c92de021a7b5ea8e (diff)
parent0383f84d88d95183638d0e227f3446974eb4e387 (diff)
downloadgitlab-ce-85ad95be741848fbf15a01789f065e001326cefa.tar.gz
Merge branch 'master' into fix/visibility-level-setting-in-forked-projects
* master: (296 commits) fox tests Don't rescue Exception, but StandardError adressing comments Update gitlab-shell documentation [ci skip] Align hash literals in IssuesFinder spec Fix tests Fix 'Attach a file' link in new tag form Add link to git-lfs client [ci skip] Do not limit workhorse POST/PUT size in NGINX added specs added spinach tests Since GitLab CI is enabled by default, remove enabling it by pushing .gitlab-ci.yml Fix tests Commits without .gitlab-ci.yml are marked as skipped Changelog entry for finding issues performance Use a JOIN in IssuableFinder#by_project Memoize IssuableFinder#projects Removed trailing whitespace from IssuableFinder Added benchmark for IssuesFinder Updated DB schema with new issues/projects indexes ... Conflicts: app/models/project.rb
Diffstat (limited to 'lib')
-rw-r--r--lib/api/api.rb1
-rw-r--r--lib/api/entities.rb56
-rw-r--r--lib/api/helpers.rb48
-rw-r--r--lib/api/projects.rb6
-rw-r--r--lib/api/repositories.rb35
-rw-r--r--lib/api/tags.rb61
-rw-r--r--lib/award_emoji.rb12
-rw-r--r--lib/backup/artifacts.rb13
-rw-r--r--lib/backup/manager.rb14
-rw-r--r--lib/ci/api/api.rb1
-rw-r--r--lib/ci/api/builds.rb100
-rw-r--r--lib/ci/api/entities.rb7
-rw-r--r--lib/ci/api/helpers.rb11
-rw-r--r--lib/ci/charts.rb3
-rw-r--r--lib/ci/gitlab_ci_yaml_processor.rb47
-rw-r--r--lib/file_streamer.rb16
-rw-r--r--lib/gitlab/backend/grack_auth.rb5
-rw-r--r--lib/gitlab/backend/shell.rb7
-rw-r--r--lib/gitlab/compare_result.rb4
-rw-r--r--lib/gitlab/current_settings.rb4
-rw-r--r--lib/gitlab/git_access.rb6
-rw-r--r--lib/gitlab/github_import/importer.rb2
-rw-r--r--lib/gitlab/google_code_import/importer.rb93
-rw-r--r--lib/gitlab/inline_diff.rb87
-rw-r--r--lib/gitlab/lfs/response.rb308
-rw-r--r--lib/gitlab/lfs/router.rb95
-rw-r--r--lib/gitlab/markdown/abstract_reference_filter.rb100
-rw-r--r--lib/gitlab/markdown/issue_reference_filter.rb63
-rw-r--r--lib/gitlab/markdown/merge_request_reference_filter.rb61
-rw-r--r--lib/gitlab/markdown/relative_link_filter.rb2
-rw-r--r--lib/gitlab/markdown/snippet_reference_filter.rb61
-rw-r--r--lib/gitlab/markdown/user_reference_filter.rb15
-rw-r--r--lib/gitlab/sherlock.rb19
-rw-r--r--lib/gitlab/sherlock/collection.rb49
-rw-r--r--lib/gitlab/sherlock/file_sample.rb31
-rw-r--r--lib/gitlab/sherlock/line_profiler.rb98
-rw-r--r--lib/gitlab/sherlock/line_sample.rb36
-rw-r--r--lib/gitlab/sherlock/location.rb26
-rw-r--r--lib/gitlab/sherlock/middleware.rb41
-rw-r--r--lib/gitlab/sherlock/query.rb114
-rw-r--r--lib/gitlab/sherlock/transaction.rb131
-rw-r--r--lib/gitlab/sql/union.rb34
-rw-r--r--lib/support/nginx/gitlab25
-rw-r--r--lib/support/nginx/gitlab-ssl25
-rw-r--r--lib/tasks/flay.rake9
-rw-r--r--lib/tasks/flog.rake25
-rw-r--r--lib/tasks/gitlab/backup.rake21
-rw-r--r--lib/tasks/grape.rake8
-rw-r--r--lib/uploaded_file.rb37
49 files changed, 1734 insertions, 339 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 40671e2517c..fe1bf8a4816 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -52,5 +52,6 @@ module API
mount Labels
mount Settings
mount Keys
+ mount Tags
end
end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 883a5e14b17..3da6bc415d6 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -62,7 +62,7 @@ module API
expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
expose :name, :name_with_namespace
expose :path, :path_with_namespace
- expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :snippets_enabled, :created_at, :last_activity_at
+ expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :created_at, :last_activity_at
expose :creator_id
expose :namespace
expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ | project, options | project.forked? }
@@ -95,25 +95,6 @@ module API
end
end
- class RepoTag < Grape::Entity
- expose :name
- expose :message do |repo_obj, _options|
- if repo_obj.respond_to?(:message)
- repo_obj.message
- else
- nil
- end
- end
-
- expose :commit do |repo_obj, options|
- if repo_obj.respond_to?(:commit)
- repo_obj.commit
- elsif options[:project]
- options[:project].repository.commit(repo_obj.target)
- end
- end
- end
-
class RepoObject < Grape::Entity
expose :name
@@ -181,7 +162,7 @@ module API
end
class MergeRequest < ProjectEntity
- expose :target_branch, :source_branch, :upvotes, :downvotes
+ expose :target_branch, :source_branch
expose :author, :assignee, using: Entities::UserBasic
expose :source_project_id, :target_project_id
expose :label_names, as: :labels
@@ -211,8 +192,6 @@ module API
expose :author, using: Entities::UserBasic
expose :created_at
expose :system?, as: :system
- expose :upvote?, as: :upvote
- expose :downvote?, as: :downvote
end
class MRNote < Grape::Entity
@@ -231,7 +210,7 @@ module API
class CommitStatus < Grape::Entity
expose :id, :sha, :ref, :status, :name, :target_url, :description,
- :created_at, :started_at, :finished_at
+ :created_at, :started_at, :finished_at, :allow_failure
expose :author, using: Entities::UserBasic
end
@@ -341,5 +320,34 @@ module API
expose :user_oauth_applications
expose :after_sign_out_path
end
+
+ class Release < Grape::Entity
+ expose :tag, :description
+ end
+
+ class RepoTag < Grape::Entity
+ expose :name
+ expose :message do |repo_obj, _options|
+ if repo_obj.respond_to?(:message)
+ repo_obj.message
+ else
+ nil
+ end
+ end
+
+ expose :commit do |repo_obj, options|
+ if repo_obj.respond_to?(:commit)
+ repo_obj.commit
+ elsif options[:project]
+ options[:project].repository.commit(repo_obj.target)
+ end
+ end
+
+ expose :release, using: Entities::Release do |repo_obj, options|
+ if options[:project]
+ options[:project].releases.find_by(tag: repo_obj.name)
+ end
+ end
+ end
end
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 652bdf9b278..92540ccf2b1 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project
end
+ def require_gitlab_workhorse!
+ unless env['HTTP_GITLAB_WORKHORSE'].present?
+ forbidden!('Request should be executed via GitLab Workhorse')
+ end
+ end
+
def can?(object, action, subject)
abilities.allowed?(object, action, subject)
end
@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
+ def file_to_large!
+ render_api_error!('413 Request Entity Too Large', 413)
+ end
+
def render_validation_error!(model)
if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +292,44 @@ module API
end
end
+ # file helpers
+
+ def uploaded_file!(field, uploads_path)
+ if params[field]
+ bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
+ return params[field]
+ end
+
+ # sanitize file paths
+ # this requires all paths to exist
+ required_attributes! %W(#{field}.path)
+ uploads_path = File.realpath(uploads_path)
+ file_path = File.realpath(params["#{field}.path"])
+ bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+
+ UploadedFile.new(
+ file_path,
+ params["#{field}.name"],
+ params["#{field}.type"] || 'application/octet-stream',
+ )
+ end
+
+ def present_file!(path, filename, content_type = 'application/octet-stream')
+ filename ||= File.basename(path)
+ header['Content-Disposition'] = "attachment; filename=#{filename}"
+ header['Content-Transfer-Encoding'] = 'binary'
+ content_type content_type
+
+ # Support download acceleration
+ case headers['X-Sendfile-Type']
+ when 'X-Sendfile'
+ header['X-Sendfile'] = path
+ body
+ else
+ file FileStreamer.new(path)
+ end
+ end
+
private
def add_pagination_headers(paginated, per_page)
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 67ee66a2058..2b4ada6e2eb 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -75,6 +75,7 @@ module API
# description (optional) - short project description
# issues_enabled (optional)
# merge_requests_enabled (optional)
+ # builds_enabled (optional)
# wiki_enabled (optional)
# snippets_enabled (optional)
# namespace_id (optional) - defaults to user namespace
@@ -90,6 +91,7 @@ module API
:description,
:issues_enabled,
:merge_requests_enabled,
+ :builds_enabled,
:wiki_enabled,
:snippets_enabled,
:namespace_id,
@@ -117,6 +119,7 @@ module API
# default_branch (optional) - 'master' by default
# issues_enabled (optional)
# merge_requests_enabled (optional)
+ # builds_enabled (optional)
# wiki_enabled (optional)
# snippets_enabled (optional)
# public (optional) - if true same as setting visibility_level = 20
@@ -132,6 +135,7 @@ module API
:default_branch,
:issues_enabled,
:merge_requests_enabled,
+ :builds_enabled,
:wiki_enabled,
:snippets_enabled,
:public,
@@ -172,6 +176,7 @@ module API
# description (optional) - short project description
# issues_enabled (optional)
# merge_requests_enabled (optional)
+ # builds_enabled (optional)
# wiki_enabled (optional)
# snippets_enabled (optional)
# public (optional) - if true same as setting visibility_level = 20
@@ -185,6 +190,7 @@ module API
:default_branch,
:issues_enabled,
:merge_requests_enabled,
+ :builds_enabled,
:wiki_enabled,
:snippets_enabled,
:public,
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 20d568cf462..d7c48639eba 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -16,41 +16,6 @@ module API
end
end
- # Get a project repository tags
- #
- # Parameters:
- # id (required) - The ID of a project
- # Example Request:
- # GET /projects/:id/repository/tags
- get ":id/repository/tags" do
- present user_project.repo.tags.sort_by(&:name).reverse,
- with: Entities::RepoTag, project: user_project
- end
-
- # Create tag
- #
- # Parameters:
- # id (required) - The ID of a project
- # tag_name (required) - The name of the tag
- # ref (required) - Create tag from commit sha or branch
- # message (optional) - Specifying a message creates an annotated tag.
- # Example Request:
- # POST /projects/:id/repository/tags
- post ':id/repository/tags' do
- authorize_push_project
- message = params[:message] || nil
- result = CreateTagService.new(user_project, current_user).
- execute(params[:tag_name], params[:ref], message)
-
- if result[:status] == :success
- present result[:tag],
- with: Entities::RepoTag,
- project: user_project
- else
- render_api_error!(result[:message], 400)
- end
- end
-
# Get a project repository tree
#
# Parameters:
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
new file mode 100644
index 00000000000..673342dd447
--- /dev/null
+++ b/lib/api/tags.rb
@@ -0,0 +1,61 @@
+module API
+ # Git Tags API
+ class Tags < Grape::API
+ before { authenticate! }
+ before { authorize! :download_code, user_project }
+
+ resource :projects do
+ # Get a project repository tags
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # Example Request:
+ # GET /projects/:id/repository/tags
+ get ":id/repository/tags" do
+ present user_project.repo.tags.sort_by(&:name).reverse,
+ with: Entities::RepoTag, project: user_project
+ end
+
+ # Create tag
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # tag_name (required) - The name of the tag
+ # ref (required) - Create tag from commit sha or branch
+ # message (optional) - Specifying a message creates an annotated tag.
+ # Example Request:
+ # POST /projects/:id/repository/tags
+ post ':id/repository/tags' do
+ authorize_push_project
+ message = params[:message] || nil
+ result = CreateTagService.new(user_project, current_user).
+ execute(params[:tag_name], params[:ref], message, params[:release_description])
+
+ if result[:status] == :success
+ present result[:tag],
+ with: Entities::RepoTag,
+ project: user_project
+ else
+ render_api_error!(result[:message], 400)
+ end
+ end
+
+ # Add release notes to tag
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # tag (required) - The name of the tag
+ # description (required) - Release notes with markdown support
+ # Example Request:
+ # PUT /projects/:id/repository/tags
+ put ':id/repository/:tag/release', requirements: { tag: /.*/ } do
+ authorize_push_project
+ required_attributes! [:description]
+ release = user_project.releases.find_or_initialize_by(tag: params[:tag])
+ release.update_attributes(description: params[:description])
+
+ present release, with: Entities::Release
+ end
+ end
+ end
+end
diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb
new file mode 100644
index 00000000000..d58a196c4ef
--- /dev/null
+++ b/lib/award_emoji.rb
@@ -0,0 +1,12 @@
+class AwardEmoji
+ EMOJI_LIST = [
+ "+1", "-1", "100", "blush", "heart", "smile", "rage",
+ "beers", "disappointed", "ok_hand",
+ "helicopter", "shit", "airplane", "alarm_clock",
+ "ambulance", "anguished", "two_hearts", "wink"
+ ]
+
+ def self.path_to_emoji_image(name)
+ "emoji/#{Emoji.emoji_filename(name)}.png"
+ end
+end
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+ class Artifacts < Files
+ def initialize
+ super('artifacts', ArtifactUploader.artifacts_path)
+ end
+
+ def create_files_dir
+ Dir.mkdir(app_files_dir, 0700)
+ end
+ end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index f011fd03de0..e7eda7c6f45 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -150,17 +150,15 @@ module Backup
private
def backup_contents
- folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
+ folders_to_backup + archives_to_backup + ["backup_information.yml"]
end
- def folders_to_backup
- folders = %w{repositories db}
-
- if ENV["SKIP"]
- return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
- end
+ def archives_to_backup
+ %w{uploads builds artifacts}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
+ end
- folders
+ def folders_to_backup
+ %w{repositories db}.reject{ |name| skipped?(name) }
end
def settings
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 0a4cbf69b63..07e68216d7f 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -27,6 +27,7 @@ module Ci
helpers Helpers
helpers ::API::Helpers
+ helpers Gitlab::CurrentSettings
mount Builds
mount Commits
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 83ca1e6481c..0a586672807 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -47,6 +47,106 @@ module Ci
build.drop
end
end
+
+ # Authorize artifacts uploading for build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # filesize (optional) - the size of uploaded file
+ # Example Request:
+ # POST /builds/:id/artifacts/authorize
+ post ":id/artifacts/authorize" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ if params[:filesize]
+ file_size = params[:filesize].to_i
+ file_to_large! unless file_size < max_artifacts_size
+ end
+
+ status 200
+ { TempPath: ArtifactUploader.artifacts_upload_path }
+ end
+
+ # Upload artifacts to build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # file (required) - The uploaded file
+ # Parameters (accelerated by GitLab Workhorse):
+ # file.path - path to locally stored body (generated by Workhorse)
+ # file.name - real filename as send in Content-Disposition
+ # file.type - real content type as send in Content-Type
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Body:
+ # The file content
+ #
+ # Example Request:
+ # POST /builds/:id/artifacts
+ post ":id/artifacts" do
+ require_gitlab_workhorse!
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ forbidden!('build is not running') unless build.running?
+
+ file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
+ file_to_large! unless file.size < max_artifacts_size
+
+ if build.update_attributes(artifacts_file: file)
+ present build, with: Entities::Build
+ else
+ render_validation_error!(build)
+ end
+ end
+
+ # Download the artifacts file from build - Runners only
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # GET /builds/:id/artifacts
+ get ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ artifacts_file = build.artifacts_file
+
+ unless artifacts_file.file_storage?
+ return redirect_to build.artifacts_file.url
+ end
+
+ unless artifacts_file.exists?
+ not_found!
+ end
+
+ present_file!(artifacts_file.path, artifacts_file.filename)
+ end
+
+ # Remove the artifacts file from build
+ #
+ # Parameters:
+ # id (required) - The ID of a build
+ # token (required) - The build authorization token
+ # Headers:
+ # BUILD-TOKEN (required) - The build authorization token, the same as token
+ # Example Request:
+ # DELETE /builds/:id/artifacts
+ delete ":id/artifacts" do
+ build = Ci::Build.find_by_id(params[:id])
+ not_found! unless build
+ authenticate_build_token!(build)
+ build.remove_artifacts_file!
+ end
end
end
end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index b80c0b8b273..750f421872d 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -11,10 +11,16 @@ module Ci
expose :builds
end
+ class ArtifactFile < Grape::Entity
+ expose :filename, :size
+ end
+
class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name
+ expose :name, :token, :stage
+
expose :options do |model|
model.options
end
@@ -24,6 +30,7 @@ module Ci
end
expose :variables
+ expose :artifacts_file, using: ArtifactFile
end
class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 7e4986b6af3..02502333756 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,6 +1,8 @@
module Ci
module API
module Helpers
+ BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+ BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60
def authenticate_runners!
@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token])
end
+ def authenticate_build_token!(build)
+ token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
+ forbidden! unless token && build.valid_token?(token)
+ end
+
def update_runner_last_contact
# Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info)
end
+
+ def max_artifacts_size
+ current_application_settings.max_artifacts_size.megabytes.to_i
+ end
end
end
end
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 915a4f526a6..5ff7407c6fe 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -60,7 +60,8 @@ module Ci
class BuildTime < Chart
def collect
- commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30)
+ commits = project.commits.last(30)
+
commits.each do |commit|
@labels << commit.short_sha
@build_times << (commit.duration / 60)
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 0f57a4f53ab..3beafcad117 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,10 +4,10 @@ module Ci
DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test'
- ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
- ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
+ ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache]
+ ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache]
- attr_reader :before_script, :image, :services, :variables, :path
+ attr_reader :before_script, :image, :services, :variables, :path, :cache
def initialize(config, path = nil)
@config = YAML.load(config)
@@ -46,6 +46,7 @@ module Ci
@services = @config[:services]
@stages = @config[:stages] || @config[:types]
@variables = @config[:variables] || {}
+ @cache = @config[:cache]
@config.except!(*ALLOWED_YAML_KEYS)
# anything that doesn't have script is considered as unknown
@@ -77,7 +78,9 @@ module Ci
when: job[:when] || 'on_success',
options: {
image: job[:image] || @image,
- services: job[:services] || @services
+ services: job[:services] || @services,
+ artifacts: job[:artifacts],
+ cache: job[:cache] || @cache,
}.compact
}
end
@@ -111,6 +114,16 @@ module Ci
raise ValidationError, "variables should be a map of key-valued strings"
end
+ if @cache
+ if @cache[:untracked] && !validate_boolean(@cache[:untracked])
+ raise ValidationError, "cache:untracked parameter should be an boolean"
+ end
+
+ if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
+ raise ValidationError, "cache:paths parameter should be an array of strings"
+ end
+ end
+
@jobs.each do |name, job|
validate_job!(name, job)
end
@@ -159,7 +172,27 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings"
end
- if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+ if job[:cache]
+ if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
+ raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
+ end
+
+ if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
+ raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
+ end
+ end
+
+ if job[:artifacts]
+ if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
+ raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
+ end
+
+ if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
+ raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
+ end
+ end
+
+ if job[:allow_failure] && !validate_boolean(job[:allow_failure])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
@@ -182,6 +215,10 @@ module Ci
value.is_a?(String) || value.is_a?(Symbol)
end
+ def validate_boolean(value)
+ value.in?([true, false])
+ end
+
def process?(only_params, except_params, ref, tag)
if only_params.present?
return false unless matching?(only_params, ref, tag)
diff --git a/lib/file_streamer.rb b/lib/file_streamer.rb
new file mode 100644
index 00000000000..4e3c6d3c773
--- /dev/null
+++ b/lib/file_streamer.rb
@@ -0,0 +1,16 @@
+class FileStreamer #:nodoc:
+ attr_reader :to_path
+
+ def initialize(path)
+ @to_path = path
+ end
+
+ # Stream the file's contents if Rack::Sendfile isn't present.
+ def each
+ File.open(to_path, 'rb') do |file|
+ while chunk = file.read(16384)
+ yield chunk
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 440ef5a3cb3..0d156047ff0 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -33,6 +33,9 @@ module Grack
auth!
+ lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call
+ return lfs_response unless lfs_response.nil?
+
if project && authorized_request?
# Tell gitlab-workhorse the request is OK, and what the GL_ID is
render_grack_auth_ok
@@ -72,7 +75,7 @@ module Grack
matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login)
if project && matched_login.present? && git_cmd == 'git-upload-pack'
- underscored_service = matched_login['s'].underscore
+ underscored_service = matched_login['s'].underscore
if Service.available_services_names.include?(underscored_service)
service_method = "#{underscored_service}_service"
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 01b8bda05c6..87ac30b5ffe 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -1,6 +1,6 @@
module Gitlab
class Shell
- class AccessDenied < StandardError; end
+ class Error < StandardError; end
class KeyAdder < Struct.new(:io)
def add_key(id, key)
@@ -36,8 +36,9 @@ module Gitlab
# import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
def import_repository(name, url)
- Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project',
- "#{name}.git", url, '240'])
+ output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240'])
+ raise Error, output unless status.zero?
+ true
end
# Move repository
diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb
index d72391dade5..0d696a1ee28 100644
--- a/lib/gitlab/compare_result.rb
+++ b/lib/gitlab/compare_result.rb
@@ -2,8 +2,8 @@ module Gitlab
class CompareResult
attr_reader :commits, :diffs
- def initialize(compare)
- @commits, @diffs = compare.commits, compare.diffs
+ def initialize(compare, diff_options = {})
+ @commits, @diffs = compare.commits, compare.diffs(nil, diff_options)
end
end
end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 0ea1b6a2f6f..2d3e32d9539 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -23,7 +23,9 @@ module Gitlab
restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
max_attachment_size: Settings.gitlab['max_attachment_size'],
session_expire_delay: Settings.gitlab['session_expire_delay'],
- import_sources: Settings.gitlab['import_sources']
+ import_sources: Settings.gitlab['import_sources'],
+ shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+ max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
)
end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index c90184d31cf..3ed1eec517c 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -13,7 +13,7 @@ module Gitlab
def user
return @user if defined?(@user)
- @user =
+ @user =
case actor
when User
actor
@@ -125,7 +125,7 @@ module Gitlab
def change_access_check(change)
oldrev, newrev, ref = change.split(' ')
- action =
+ action =
if project.protected_branch?(branch_name(ref))
protected_branch_action(oldrev, newrev, branch_name(ref))
elsif protected_tag?(tag_name(ref))
@@ -148,7 +148,7 @@ module Gitlab
build_status_object(false, "You are not allowed to change existing tags on this project.")
else # :push_code
build_status_object(false, "You are not allowed to push code to this project.")
- end
+ end
return status
end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index bd7340a80f1..b5720f6e2cb 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -19,7 +19,7 @@ module Gitlab
if issue.pull_request.nil?
body = @formatter.author_line(issue.user.login)
- body += issue.body
+ body += issue.body || ""
if issue.comments > 0
body += @formatter.comments_header
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 03c410726a5..87fee28dc01 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -30,7 +30,7 @@ module Gitlab
def user_map
@user_map ||= begin
- user_map = Hash.new do |hash, user|
+ user_map = Hash.new do |hash, user|
# Replace ... by \.\.\., so `johnsm...@gmail.com` isn't autolinked.
Client.mask_email(user).sub("...", "\\.\\.\\.")
end
@@ -76,18 +76,7 @@ module Gitlab
attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"])
body = format_issue_body(author, date, content, attachments)
-
- labels = []
- raw_issue["labels"].each do |label|
- name = nice_label_name(label)
- labels << name
-
- unless @known_labels.include?(name)
- create_label(name)
- @known_labels << name
- end
- end
- labels << nice_status_name(raw_issue["status"])
+ labels = import_issue_labels(raw_issue)
assignee_id = nil
if raw_issue.has_key?("owner")
@@ -110,6 +99,7 @@ module Gitlab
assignee_id: assignee_id,
state: raw_issue["state"] == "closed" ? "closed" : "opened"
)
+
issue.add_labels_by_names(labels)
if issue.iid != raw_issue["id"]
@@ -120,6 +110,23 @@ module Gitlab
end
end
+ def import_issue_labels(raw_issue)
+ labels = []
+
+ raw_issue["labels"].each do |label|
+ name = nice_label_name(label)
+ labels << name
+
+ unless @known_labels.include?(name)
+ create_label(name)
+ @known_labels << name
+ end
+ end
+
+ labels << nice_status_name(raw_issue["status"])
+ labels
+ end
+
def import_issue_comments(issue, comments)
Note.transaction do
while raw_comment = comments.shift
@@ -172,7 +179,7 @@ module Gitlab
"#5cb85c"
when "Status: Started"
"#8e44ad"
-
+
when "Priority: Critical"
"#ffcfcf"
when "Priority: High"
@@ -181,7 +188,7 @@ module Gitlab
"#fff5cc"
when "Priority: Low"
"#cfe9ff"
-
+
when "Type: Defect"
"#d9534f"
when "Type: Enhancement"
@@ -249,8 +256,8 @@ module Gitlab
end
if raw_updates.has_key?("cc")
- cc = raw_updates["cc"].map do |l|
- deleted = l.start_with?("-")
+ cc = raw_updates["cc"].map do |l|
+ deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = user_map[l]
l = "~~#{l}~~" if deleted
@@ -261,8 +268,8 @@ module Gitlab
end
if raw_updates.has_key?("labels")
- labels = raw_updates["labels"].map do |l|
- deleted = l.start_with?("-")
+ labels = raw_updates["labels"].map do |l|
+ deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = nice_label_name(l)
l = "~~#{l}~~" if deleted
@@ -278,45 +285,39 @@ module Gitlab
if raw_updates.has_key?("blockedOn")
blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
- name, id = raw_blocked_on.split(":", 2)
-
- deleted = name.start_with?("-")
- name = name[1..-1] if deleted
-
- text =
- if name == project.import_source
- "##{id}"
- else
- "#{project.namespace.path}/#{name}##{id}"
- end
- text = "~~#{text}~~" if deleted
- text
+ format_blocking_updates(raw_blocked_on)
end
+
updates << "*Blocked on: #{blocked_ons.join(", ")}*"
end
if raw_updates.has_key?("blocking")
blockings = raw_updates["blocking"].map do |raw_blocked_on|
- name, id = raw_blocked_on.split(":", 2)
-
- deleted = name.start_with?("-")
- name = name[1..-1] if deleted
-
- text =
- if name == project.import_source
- "##{id}"
- else
- "#{project.namespace.path}/#{name}##{id}"
- end
- text = "~~#{text}~~" if deleted
- text
+ format_blocking_updates(raw_blocked_on)
end
+
updates << "*Blocking: #{blockings.join(", ")}*"
end
updates
end
+ def format_blocking_updates(raw_blocked_on)
+ name, id = raw_blocked_on.split(":", 2)
+
+ deleted = name.start_with?("-")
+ name = name[1..-1] if deleted
+
+ text =
+ if name == project.import_source
+ "##{id}"
+ else
+ "#{project.namespace.path}/#{name}##{id}"
+ end
+ text = "~~#{text}~~" if deleted
+ text
+ end
+
def format_attachments(issue_id, comment_id, raw_attachments)
return [] unless raw_attachments
@@ -325,7 +326,7 @@ module Gitlab
filename = attachment["fileName"]
link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{filename}"
-
+
text = "[#{filename}](#{link})"
text = "!#{text}" if filename =~ /\.(png|jpg|jpeg|gif|bmp|tiff)\z/i
text
diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb
index 99e7b529ba9..44507bde25d 100644
--- a/lib/gitlab/inline_diff.rb
+++ b/lib/gitlab/inline_diff.rb
@@ -11,48 +11,71 @@ module Gitlab
indexes.each do |index|
first_line = diff_arr[index+1]
second_line = diff_arr[index+2]
- max_length = [first_line.size, second_line.size].max
# Skip inline diff if empty line was replaced with content
next if first_line == "-\n"
- first_the_same_symbols = 0
- (0..max_length + 1).each do |i|
- first_the_same_symbols = i - 1
- if first_line[i] != second_line[i] && i > 0
- break
- end
- end
+ first_token = find_first_token(first_line, second_line)
+ apply_first_token(diff_arr, index, first_token)
+
+ last_token = find_last_token(first_line, second_line, first_token)
+ apply_last_token(diff_arr, index, last_token)
+ end
+
+ diff_arr
+ end
+
+ def apply_first_token(diff_arr, index, first_token)
+ start = first_token + START
+
+ if first_token.empty?
+ # In case if we remove string of spaces in commit
+ diff_arr[index+1].sub!("-", "-" => "-#{START}")
+ diff_arr[index+2].sub!("+", "+" => "+#{START}")
+ else
+ diff_arr[index+1].sub!(first_token, first_token => start)
+ diff_arr[index+2].sub!(first_token, first_token => start)
+ end
+ end
- first_token = first_line[0..first_the_same_symbols][1..-1]
- start = first_token + START
+ def apply_last_token(diff_arr, index, last_token)
+ # This is tricky: escape backslashes so that `sub` doesn't interpret them
+ # as backreferences. Regexp.escape does NOT do the right thing.
+ replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
+ diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
+ diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
+ end
+
+ def find_first_token(first_line, second_line)
+ max_length = [first_line.size, second_line.size].max
+ first_the_same_symbols = 0
+
+ (0..max_length + 1).each do |i|
+ first_the_same_symbols = i - 1
- if first_token.empty?
- # In case if we remove string of spaces in commit
- diff_arr[index+1].sub!("-", "-" => "-#{START}")
- diff_arr[index+2].sub!("+", "+" => "+#{START}")
- else
- diff_arr[index+1].sub!(first_token, first_token => start)
- diff_arr[index+2].sub!(first_token, first_token => start)
+ if first_line[i] != second_line[i] && i > 0
+ break
end
+ end
+
+ first_line[0..first_the_same_symbols][1..-1]
+ end
+
+ def find_last_token(first_line, second_line, first_token)
+ max_length = [first_line.size, second_line.size].max
+ last_the_same_symbols = 0
+
+ (1..max_length + 1).each do |i|
+ last_the_same_symbols = -i
+ shortest_line = second_line.size > first_line.size ? first_line : second_line
- last_the_same_symbols = 0
- (1..max_length + 1).each do |i|
- last_the_same_symbols = -i
- shortest_line = second_line.size > first_line.size ? first_line : second_line
- if ( first_line[-i] != second_line[-i] ) || "#{first_token}#{START}".size == shortest_line[1..-i].size
- break
- end
+ if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size
+ break
end
- last_the_same_symbols += 1
- last_token = first_line[last_the_same_symbols..-1]
- # This is tricky: escape backslashes so that `sub` doesn't interpret them
- # as backreferences. Regexp.escape does NOT do the right thing.
- replace_token = FINISH + last_token.gsub(/\\/, '\&\&')
- diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
- diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token)
end
- diff_arr
+
+ last_the_same_symbols += 1
+ first_line[last_the_same_symbols..-1]
end
def _indexes_of_changed_lines(diff_arr)
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
new file mode 100644
index 00000000000..4202c786466
--- /dev/null
+++ b/lib/gitlab/lfs/response.rb
@@ -0,0 +1,308 @@
+module Gitlab
+ module Lfs
+ class Response
+
+ def initialize(project, user, request)
+ @origin_project = project
+ @project = storage_project(project)
+ @user = user
+ @env = request.env
+ @request = request
+ end
+
+ # Return a response for a download request
+ # Can be a response to:
+ # Request from a user to get the file
+ # Request from gitlab-workhorse which file to serve to the user
+ def render_download_hypermedia_response(oid)
+ render_response_to_download do
+ if check_download_accept_header?
+ render_lfs_download_hypermedia(oid)
+ else
+ render_not_found
+ end
+ end
+ end
+
+ def render_download_object_response(oid)
+ render_response_to_download do
+ if check_download_sendfile_header? && check_download_accept_header?
+ render_lfs_sendfile(oid)
+ else
+ render_not_found
+ end
+ end
+ end
+
+ def render_lfs_api_auth
+ render_response_to_push do
+ request_body = JSON.parse(@request.body.read)
+ return render_not_found if request_body.empty? || request_body['objects'].empty?
+
+ response = build_response(request_body['objects'])
+ [
+ 200,
+ {
+ "Content-Type" => "application/json; charset=utf-8",
+ "Cache-Control" => "private",
+ },
+ [JSON.dump(response)]
+ ]
+ end
+ end
+
+ def render_storage_upload_authorize_response(oid, size)
+ render_response_to_push do
+ [
+ 200,
+ { "Content-Type" => "application/json; charset=utf-8" },
+ [JSON.dump({
+ 'StoreLFSPath' => "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+ 'LfsOid' => oid,
+ 'LfsSize' => size
+ })]
+ ]
+ end
+ end
+
+ def render_storage_upload_store_response(oid, size, tmp_file_name)
+ render_response_to_push do
+ render_lfs_upload_ok(oid, size, tmp_file_name)
+ end
+ end
+
+ private
+
+ def render_not_enabled
+ [
+ 501,
+ {
+ "Content-Type" => "application/vnd.git-lfs+json",
+ },
+ [JSON.dump({
+ 'message' => 'Git LFS is not enabled on this GitLab server, contact your admin.',
+ 'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+ })]
+ ]
+ end
+
+ def render_unauthorized
+ [
+ 401,
+ {
+ 'Content-Type' => 'text/plain'
+ },
+ ['Unauthorized']
+ ]
+ end
+
+ def render_not_found
+ [
+ 404,
+ {
+ "Content-Type" => "application/vnd.git-lfs+json"
+ },
+ [JSON.dump({
+ 'message' => 'Not found.',
+ 'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+ })]
+ ]
+ end
+
+ def render_forbidden
+ [
+ 403,
+ {
+ "Content-Type" => "application/vnd.git-lfs+json"
+ },
+ [JSON.dump({
+ 'message' => 'Access forbidden. Check your access level.',
+ 'documentation_url' => "#{Gitlab.config.gitlab.url}/help",
+ })]
+ ]
+ end
+
+ def render_lfs_sendfile(oid)
+ return render_not_found unless oid.present?
+
+ lfs_object = object_for_download(oid)
+
+ if lfs_object && lfs_object.file.exists?
+ [
+ 200,
+ {
+ # GitLab-workhorse will forward Content-Type header
+ "Content-Type" => "application/octet-stream",
+ "X-Sendfile" => lfs_object.file.path
+ },
+ []
+ ]
+ else
+ render_not_found
+ end
+ end
+
+ def render_lfs_download_hypermedia(oid)
+ return render_not_found unless oid.present?
+
+ lfs_object = object_for_download(oid)
+ if lfs_object
+ [
+ 200,
+ { "Content-Type" => "application/vnd.git-lfs+json" },
+ [JSON.dump(download_hypermedia(oid))]
+ ]
+ else
+ render_not_found
+ end
+ end
+
+ def render_lfs_upload_ok(oid, size, tmp_file)
+ if store_file(oid, size, tmp_file)
+ [
+ 200,
+ {
+ 'Content-Type' => 'text/plain',
+ 'Content-Length' => 0
+ },
+ []
+ ]
+ else
+ [
+ 422,
+ { 'Content-Type' => 'text/plain' },
+ ["Unprocessable entity"]
+ ]
+ end
+ end
+
+ def render_response_to_download
+ return render_not_enabled unless Gitlab.config.lfs.enabled
+
+ unless @project.public?
+ return render_unauthorized unless @user
+ return render_forbidden unless user_can_fetch?
+ end
+
+ yield
+ end
+
+ def render_response_to_push
+ return render_not_enabled unless Gitlab.config.lfs.enabled
+ return render_unauthorized unless @user
+ return render_forbidden unless user_can_push?
+
+ yield
+ end
+
+ def check_download_sendfile_header?
+ @env['HTTP_X_SENDFILE_TYPE'].to_s == "X-Sendfile"
+ end
+
+ def check_download_accept_header?
+ @env['HTTP_ACCEPT'].to_s == "application/vnd.git-lfs+json; charset=utf-8"
+ end
+
+ def user_can_fetch?
+ # Check user access against the project they used to initiate the pull
+ @user.can?(:download_code, @origin_project)
+ end
+
+ def user_can_push?
+ # Check user access against the project they used to initiate the push
+ @user.can?(:push_code, @origin_project)
+ end
+
+ def storage_project(project)
+ if project.forked?
+ project.forked_from_project
+ else
+ project
+ end
+ end
+
+ def store_file(oid, size, tmp_file)
+ tmp_file_path = File.join("#{Gitlab.config.lfs.storage_path}/tmp/upload", tmp_file)
+
+ object = LfsObject.find_or_create_by(oid: oid, size: size)
+ if object.file.exists?
+ success = true
+ else
+ success = move_tmp_file_to_storage(object, tmp_file_path)
+ end
+
+ if success
+ success = link_to_project(object)
+ end
+
+ success
+ ensure
+ # Ensure that the tmp file is removed
+ FileUtils.rm_f(tmp_file_path)
+ end
+
+ def object_for_download(oid)
+ @project.lfs_objects.find_by(oid: oid)
+ end
+
+ def move_tmp_file_to_storage(object, path)
+ File.open(path) do |f|
+ object.file = f
+ end
+
+ object.file.store!
+ object.save
+ end
+
+ def link_to_project(object)
+ if object && !object.projects.exists?(@project)
+ object.projects << @project
+ object.save
+ end
+ end
+
+ def select_existing_objects(objects)
+ objects_oids = objects.map { |o| o['oid'] }
+ @project.lfs_objects.where(oid: objects_oids).pluck(:oid).to_set
+ end
+
+ def build_response(objects)
+ selected_objects = select_existing_objects(objects)
+
+ upload_hypermedia(objects, selected_objects)
+ end
+
+ def download_hypermedia(oid)
+ {
+ '_links' => {
+ 'download' =>
+ {
+ 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{oid}",
+ 'header' => {
+ 'Accept' => "application/vnd.git-lfs+json; charset=utf-8",
+ 'Authorization' => @env['HTTP_AUTHORIZATION']
+ }.compact
+ }
+ }
+ }
+ end
+
+ def upload_hypermedia(all_objects, existing_objects)
+ all_objects.each do |object|
+ object['_links'] = hypermedia_links(object) unless existing_objects.include?(object['oid'])
+ end
+
+ { 'objects' => all_objects }
+ end
+
+ def hypermedia_links(object)
+ {
+ "upload" => {
+ 'href' => "#{@origin_project.http_url_to_repo}/gitlab-lfs/objects/#{object['oid']}/#{object['size']}",
+ 'header' => { 'Authorization' => @env['HTTP_AUTHORIZATION'] }
+ }.compact
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
new file mode 100644
index 00000000000..4809e834984
--- /dev/null
+++ b/lib/gitlab/lfs/router.rb
@@ -0,0 +1,95 @@
+module Gitlab
+ module Lfs
+ class Router
+ def initialize(project, user, request)
+ @project = project
+ @user = user
+ @env = request.env
+ @request = request
+ end
+
+ def try_call
+ return unless @request && @request.path.present?
+
+ case @request.request_method
+ when 'GET'
+ get_response
+ when 'POST'
+ post_response
+ when 'PUT'
+ put_response
+ else
+ nil
+ end
+ end
+
+ private
+
+ def get_response
+ path_match = @request.path.match(/\/(info\/lfs|gitlab-lfs)\/objects\/([0-9a-f]{64})$/)
+ return nil unless path_match
+
+ oid = path_match[2]
+ return nil unless oid
+
+ case path_match[1]
+ when "info/lfs"
+ lfs.render_download_hypermedia_response(oid)
+ when "gitlab-lfs"
+ lfs.render_download_object_response(oid)
+ else
+ nil
+ end
+ end
+
+ def post_response
+ post_path = @request.path.match(/\/info\/lfs\/objects(\/batch)?$/)
+ return nil unless post_path
+
+ # Check for Batch API
+ if post_path[0].ends_with?("/info/lfs/objects/batch")
+ lfs.render_lfs_api_auth
+ else
+ nil
+ end
+ end
+
+ def put_response
+ object_match = @request.path.match(/\/gitlab-lfs\/objects\/([0-9a-f]{64})\/([0-9]+)(|\/authorize){1}$/)
+ return nil if object_match.nil?
+
+ oid = object_match[1]
+ size = object_match[2].try(:to_i)
+ return nil if oid.nil? || size.nil?
+
+ # GitLab-workhorse requests
+ # 1. Try to authorize the request
+ # 2. send a request with a header containing the name of the temporary file
+ if object_match[3] && object_match[3] == '/authorize'
+ lfs.render_storage_upload_authorize_response(oid, size)
+ else
+ tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP'])
+ return nil unless tmp_file_name
+
+ lfs.render_storage_upload_store_response(oid, size, tmp_file_name)
+ end
+ end
+
+ def lfs
+ return unless @project
+
+ Gitlab::Lfs::Response.new(@project, @user, @request)
+ end
+
+ def sanitize_tmp_filename(name)
+ if name.present?
+ name.gsub!(/^.*(\\|\/)/, '')
+ name = name.match(/[0-9a-f]{73}/)
+ name[0] if name
+ else
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/markdown/abstract_reference_filter.rb b/lib/gitlab/markdown/abstract_reference_filter.rb
new file mode 100644
index 00000000000..fd5b7eb9332
--- /dev/null
+++ b/lib/gitlab/markdown/abstract_reference_filter.rb
@@ -0,0 +1,100 @@
+require 'gitlab/markdown'
+
+module Gitlab
+ module Markdown
+ # Issues, Snippets and Merge Requests shares similar functionality in refernce filtering.
+ # All this functionality moved to this class
+ class AbstractReferenceFilter < ReferenceFilter
+ include CrossProjectReference
+
+ def self.object_class
+ # Implement in child class
+ # Example: MergeRequest
+ end
+
+ def self.object_name
+ object_class.name.underscore
+ end
+
+ def self.object_sym
+ object_name.to_sym
+ end
+
+ def self.data_reference
+ "data-#{object_name.dasherize}"
+ end
+
+ # Public: Find references in text (like `!123` for merge requests)
+ #
+ # AnyReferenceFilter.references_in(text) do |match, object|
+ # "<a href=...>PREFIX#{object}</a>"
+ # end
+ #
+ # PREFIX - symbol that detects reference (like ! for merge requests)
+ # object - reference object (snippet, merget request etc)
+ # text - String text to search.
+ #
+ # Yields the String match, the Integer referenced object ID, and an optional String
+ # of the external project reference.
+ #
+ # Returns a String replaced with the return of the block.
+ def self.references_in(text)
+ text.gsub(object_class.reference_pattern) do |match|
+ yield match, $~[object_sym].to_i, $~[:project]
+ end
+ end
+
+ def self.referenced_by(node)
+ { object_sym => LazyReference.new(object_class, node.attr(data_reference)) }
+ end
+
+ delegate :object_class, :object_sym, :references_in, to: :class
+
+ def find_object(project, id)
+ # Implement in child class
+ # Example: project.merge_requests.find
+ end
+
+ def url_for_object(object, project)
+ # Implement in child class
+ # Example: project_merge_request_url
+ end
+
+ def call
+ replace_text_nodes_matching(object_class.reference_pattern) do |content|
+ object_link_filter(content)
+ end
+ end
+
+ # Replace references (like `!123` for merge requests) in text with links
+ # to the referenced object's details page.
+ #
+ # text - String text to replace references in.
+ #
+ # Returns a String with references replaced with links. All links
+ # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
+ def object_link_filter(text)
+ references_in(text) do |match, id, project_ref|
+ project = project_from_ref(project_ref)
+
+ if project && object = find_object(project, id)
+ title = escape_once("#{object_title}: #{object.title}")
+ klass = reference_class(object_sym)
+ data = data_attribute(project: project.id, object_sym => object.id)
+ url = url_for_object(object, project)
+
+ %(<a href="#{url}" #{data}
+ title="#{title}"
+ class="#{klass}">#{match}</a>)
+ else
+ match
+ end
+ end
+ end
+
+ def object_title
+ object_class.name.titleize
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb
index 481d282f7b1..1ed69e2f431 100644
--- a/lib/gitlab/markdown/issue_reference_filter.rb
+++ b/lib/gitlab/markdown/issue_reference_filter.rb
@@ -6,66 +6,17 @@ module Gitlab
# issues that do not exist are ignored.
#
# This filter supports cross-project references.
- class IssueReferenceFilter < ReferenceFilter
- include CrossProjectReference
-
- # Public: Find `#123` issue references in text
- #
- # IssueReferenceFilter.references_in(text) do |match, issue, project_ref|
- # "<a href=...>##{issue}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, the Integer issue ID, and an optional String of
- # the external project reference.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text)
- text.gsub(Issue.reference_pattern) do |match|
- yield match, $~[:issue].to_i, $~[:project]
- end
- end
-
- def self.referenced_by(node)
- { issue: LazyReference.new(Issue, node.attr("data-issue")) }
- end
-
- def call
- replace_text_nodes_matching(Issue.reference_pattern) do |content|
- issue_link_filter(content)
- end
+ class IssueReferenceFilter < AbstractReferenceFilter
+ def self.object_class
+ Issue
end
- # Replace `#123` issue references in text with links to the referenced
- # issue's details page.
- #
- # text - String text to replace references in.
- #
- # Returns a String with `#123` references replaced with links. All links
- # have `gfm` and `gfm-issue` class names attached for styling.
- def issue_link_filter(text)
- self.class.references_in(text) do |match, id, project_ref|
- project = self.project_from_ref(project_ref)
-
- if project && issue = project.get_issue(id)
- url = url_for_issue(id, project, only_path: context[:only_path])
-
- title = escape_once("Issue: #{issue.title}")
- klass = reference_class(:issue)
- data = data_attribute(project: project.id, issue: issue.id)
-
- %(<a href="#{url}" #{data}
- title="#{title}"
- class="#{klass}">#{match}</a>)
- else
- match
- end
- end
+ def find_object(project, id)
+ project.get_issue(id)
end
- def url_for_issue(*args)
- IssuesHelper.url_for_issue(*args)
+ def url_for_object(issue, project)
+ IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
end
end
end
diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb
index 5bc63269808..1f47f03c94e 100644
--- a/lib/gitlab/markdown/merge_request_reference_filter.rb
+++ b/lib/gitlab/markdown/merge_request_reference_filter.rb
@@ -6,65 +6,16 @@ module Gitlab
# to merge requests that do not exist are ignored.
#
# This filter supports cross-project references.
- class MergeRequestReferenceFilter < ReferenceFilter
- include CrossProjectReference
-
- # Public: Find `!123` merge request references in text
- #
- # MergeRequestReferenceFilter.references_in(text) do |match, merge_request, project_ref|
- # "<a href=...>##{merge_request}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, the Integer merge request ID, and an optional
- # String of the external project reference.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text)
- text.gsub(MergeRequest.reference_pattern) do |match|
- yield match, $~[:merge_request].to_i, $~[:project]
- end
- end
-
- def self.referenced_by(node)
- { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) }
- end
-
- def call
- replace_text_nodes_matching(MergeRequest.reference_pattern) do |content|
- merge_request_link_filter(content)
- end
+ class MergeRequestReferenceFilter < AbstractReferenceFilter
+ def self.object_class
+ MergeRequest
end
- # Replace `!123` merge request references in text with links to the
- # referenced merge request's details page.
- #
- # text - String text to replace references in.
- #
- # Returns a String with `!123` references replaced with links. All links
- # have `gfm` and `gfm-merge_request` class names attached for styling.
- def merge_request_link_filter(text)
- self.class.references_in(text) do |match, id, project_ref|
- project = self.project_from_ref(project_ref)
-
- if project && merge_request = project.merge_requests.find_by(iid: id)
- title = escape_once("Merge Request: #{merge_request.title}")
- klass = reference_class(:merge_request)
- data = data_attribute(project: project.id, merge_request: merge_request.id)
-
- url = url_for_merge_request(merge_request, project)
-
- %(<a href="#{url}" #{data}
- title="#{title}"
- class="#{klass}">#{match}</a>)
- else
- match
- end
- end
+ def find_object(project, id)
+ project.merge_requests.find_by(iid: id)
end
- def url_for_merge_request(mr, project)
+ def url_for_object(mr, project)
h = Gitlab::Application.routes.url_helpers
h.namespace_project_merge_request_url(project.namespace, project, mr,
only_path: context[:only_path])
diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb
index 6ee3d1ce039..632be4d7542 100644
--- a/lib/gitlab/markdown/relative_link_filter.rb
+++ b/lib/gitlab/markdown/relative_link_filter.rb
@@ -51,7 +51,7 @@ module Gitlab
relative_url_root,
context[:project].path_with_namespace,
path_type(file_path),
- ref || 'master', # assume that if no ref exists we can point to master
+ ref || context[:project].default_branch, # if no ref exists, point to the default branch
file_path
].compact.join('/').squeeze('/').chomp('/')
diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb
index f783f951711..f7bd07c2a34 100644
--- a/lib/gitlab/markdown/snippet_reference_filter.rb
+++ b/lib/gitlab/markdown/snippet_reference_filter.rb
@@ -6,65 +6,16 @@ module Gitlab
# snippets that do not exist are ignored.
#
# This filter supports cross-project references.
- class SnippetReferenceFilter < ReferenceFilter
- include CrossProjectReference
-
- # Public: Find `$123` snippet references in text
- #
- # SnippetReferenceFilter.references_in(text) do |match, snippet|
- # "<a href=...>$#{snippet}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, the Integer snippet ID, and an optional String
- # of the external project reference.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text)
- text.gsub(Snippet.reference_pattern) do |match|
- yield match, $~[:snippet].to_i, $~[:project]
- end
- end
-
- def self.referenced_by(node)
- { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) }
- end
-
- def call
- replace_text_nodes_matching(Snippet.reference_pattern) do |content|
- snippet_link_filter(content)
- end
+ class SnippetReferenceFilter < AbstractReferenceFilter
+ def self.object_class
+ Snippet
end
- # Replace `$123` snippet references in text with links to the referenced
- # snippets's details page.
- #
- # text - String text to replace references in.
- #
- # Returns a String with `$123` references replaced with links. All links
- # have `gfm` and `gfm-snippet` class names attached for styling.
- def snippet_link_filter(text)
- self.class.references_in(text) do |match, id, project_ref|
- project = self.project_from_ref(project_ref)
-
- if project && snippet = project.snippets.find_by(id: id)
- title = escape_once("Snippet: #{snippet.title}")
- klass = reference_class(:snippet)
- data = data_attribute(project: project.id, snippet: snippet.id)
-
- url = url_for_snippet(snippet, project)
-
- %(<a href="#{url}" #{data}
- title="#{title}"
- class="#{klass}">#{match}</a>)
- else
- match
- end
- end
+ def find_object(project, id)
+ project.snippets.find_by(id: id)
end
- def url_for_snippet(snippet, project)
+ def url_for_object(snippet, project)
h = Gitlab::Application.routes.url_helpers
h.namespace_project_snippet_url(project.namespace, project, snippet,
only_path: context[:only_path])
diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb
index 2a594e1662e..ab5e1f6fe9e 100644
--- a/lib/gitlab/markdown/user_reference_filter.rb
+++ b/lib/gitlab/markdown/user_reference_filter.rb
@@ -85,13 +85,12 @@ module Gitlab
def link_to_all
project = context[:project]
-
url = urls.namespace_project_url(project.namespace, project,
only_path: context[:only_path])
data = data_attribute(project: project.id)
-
text = User.reference_prefix + 'all'
- %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
+
+ link_tag(url, data, text)
end
def link_to_namespace(namespace)
@@ -105,16 +104,20 @@ module Gitlab
def link_to_group(group, namespace)
url = urls.group_url(group, only_path: context[:only_path])
data = data_attribute(group: namespace.id)
-
text = Group.reference_prefix + group
- %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
+
+ link_tag(url, data, text)
end
def link_to_user(user, namespace)
url = urls.user_url(user, only_path: context[:only_path])
data = data_attribute(user: namespace.owner_id)
-
text = User.reference_prefix + user
+
+ link_tag(url, data, text)
+ end
+
+ def link_tag(url, data, text)
%(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>)
end
end
diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb
new file mode 100644
index 00000000000..6360527a7aa
--- /dev/null
+++ b/lib/gitlab/sherlock.rb
@@ -0,0 +1,19 @@
+require 'securerandom'
+
+module Gitlab
+ module Sherlock
+ @collection = Collection.new
+
+ class << self
+ attr_reader :collection
+ end
+
+ def self.enabled?
+ Rails.env.development? && !!ENV['ENABLE_SHERLOCK']
+ end
+
+ def self.enable_line_profiler?
+ RUBY_ENGINE == 'ruby'
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb
new file mode 100644
index 00000000000..66bd6258521
--- /dev/null
+++ b/lib/gitlab/sherlock/collection.rb
@@ -0,0 +1,49 @@
+module Gitlab
+ module Sherlock
+ # A collection of transactions recorded by Sherlock.
+ #
+ # Method calls for this class are synchronized using a mutex to allow
+ # sharing of a single Collection instance between threads (e.g. when using
+ # Puma as a webserver).
+ class Collection
+ include Enumerable
+
+ def initialize
+ @transactions = []
+ @mutex = Mutex.new
+ end
+
+ def add(transaction)
+ synchronize { @transactions << transaction }
+ end
+
+ alias_method :<<, :add
+
+ def each(&block)
+ synchronize { @transactions.each(&block) }
+ end
+
+ def clear
+ synchronize { @transactions.clear }
+ end
+
+ def empty?
+ synchronize { @transactions.empty? }
+ end
+
+ def find_transaction(id)
+ find { |trans| trans.id == id }
+ end
+
+ def newest_first
+ sort { |a, b| b.finished_at <=> a.finished_at }
+ end
+
+ private
+
+ def synchronize(&block)
+ @mutex.synchronize(&block)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
new file mode 100644
index 00000000000..8a3e1a5e5bf
--- /dev/null
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -0,0 +1,31 @@
+module Gitlab
+ module Sherlock
+ class FileSample
+ attr_reader :id, :file, :line_samples, :events, :duration
+
+ # file - The full path to the file this sample belongs to.
+ # line_samples - An array of LineSample objects.
+ # duration - The total execution time in milliseconds.
+ # events - The total amount of events.
+ def initialize(file, line_samples, duration, events)
+ @id = SecureRandom.uuid
+ @file = file
+ @line_samples = line_samples
+ @duration = duration
+ @events = events
+ end
+
+ def relative_path
+ @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '')
+ end
+
+ def to_param
+ @id
+ end
+
+ def source
+ @source ||= File.read(@file)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
new file mode 100644
index 00000000000..aa1468bff6b
--- /dev/null
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -0,0 +1,98 @@
+module Gitlab
+ module Sherlock
+ # Class for profiling code on a per line basis.
+ #
+ # The LineProfiler class can be used to profile code on per line basis
+ # without littering your code with Ruby implementation specific profiling
+ # methods.
+ #
+ # This profiler only includes samples taking longer than a given threshold
+ # and those that occur in the actual application (e.g. files from Gems are
+ # ignored).
+ class LineProfiler
+ # The minimum amount of time that has to be spent in a file for it to be
+ # included in a list of samples.
+ MINIMUM_DURATION = 10.0
+
+ # Profiles the given block.
+ #
+ # Example:
+ #
+ # profiler = LineProfiler.new
+ #
+ # retval, samples = profiler.profile do
+ # "cats are amazing"
+ # end
+ #
+ # retval # => "cats are amazing"
+ # samples # => [#<Gitlab::Sherlock::FileSample ...>, ...]
+ #
+ # Returns an Array containing the block's return value and an Array of
+ # FileSample objects.
+ def profile(&block)
+ if mri?
+ profile_mri(&block)
+ else
+ raise NotImplementedError,
+ 'Line profiling is not supported on this platform'
+ end
+ end
+
+ # Profiles the given block using rblineprof (MRI only).
+ def profile_mri
+ require 'rblineprof'
+
+ retval = nil
+ samples = lineprof(/^#{Rails.root.to_s}/) { retval = yield }
+
+ file_samples = aggregate_rblineprof(samples)
+
+ [retval, file_samples]
+ end
+
+ # Returns an Array of file samples based on the output of rblineprof.
+ #
+ # lineprof_stats - A Hash containing rblineprof statistics on a per file
+ # basis.
+ #
+ # Returns an Array of FileSample objects.
+ def aggregate_rblineprof(lineprof_stats)
+ samples = []
+
+ lineprof_stats.each do |(file, stats)|
+ source_lines = File.read(file).each_line.to_a
+ line_samples = []
+
+ total_duration = microsec_to_millisec(stats[0][0])
+ total_events = stats[0][2]
+
+ next if total_duration <= MINIMUM_DURATION
+
+ stats[1..-1].each_with_index do |data, index|
+ next unless source_lines[index]
+
+ duration = microsec_to_millisec(data[0])
+ events = data[2]
+
+ line_samples << LineSample.new(duration, events)
+ end
+
+ samples << FileSample.
+ new(file, line_samples, total_duration, total_events)
+ end
+
+ samples
+ end
+
+ private
+
+ def microsec_to_millisec(microsec)
+ microsec / 1000.0
+ end
+
+ def mri?
+ RUBY_ENGINE == 'ruby'
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb
new file mode 100644
index 00000000000..eb1948eb6d6
--- /dev/null
+++ b/lib/gitlab/sherlock/line_sample.rb
@@ -0,0 +1,36 @@
+module Gitlab
+ module Sherlock
+ class LineSample
+ attr_reader :duration, :events
+
+ # duration - The execution time in milliseconds.
+ # events - The amount of events.
+ def initialize(duration, events)
+ @duration = duration
+ @events = events
+ end
+
+ # Returns the sample duration percentage relative to the given duration.
+ #
+ # Example:
+ #
+ # sample.duration # => 150
+ # sample.percentage_of(1500) # => 10.0
+ #
+ # total_duration - The total duration to compare with.
+ #
+ # Returns a float
+ def percentage_of(total_duration)
+ (duration.to_f / total_duration) * 100.0
+ end
+
+ # Returns true if the current sample takes up the majority of the given
+ # duration.
+ #
+ # total_duration - The total duration to compare with.
+ def majority_of?(total_duration)
+ percentage_of(total_duration) >= 30
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb
new file mode 100644
index 00000000000..5ac265618ad
--- /dev/null
+++ b/lib/gitlab/sherlock/location.rb
@@ -0,0 +1,26 @@
+module Gitlab
+ module Sherlock
+ class Location
+ attr_reader :path, :line
+
+ SHERLOCK_DIR = File.dirname(__FILE__)
+
+ # Creates a new Location from a `Thread::Backtrace::Location`.
+ def self.from_ruby_location(location)
+ new(location.path, location.lineno)
+ end
+
+ # path - The full path of the frame as a String.
+ # line - The line number of the frame as a Fixnum.
+ def initialize(path, line)
+ @path = path
+ @line = line
+ end
+
+ # Returns true if the current frame originated from the application.
+ def application?
+ @path.start_with?(Rails.root.to_s) && !path.start_with?(SHERLOCK_DIR)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
new file mode 100644
index 00000000000..687332fc5fc
--- /dev/null
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -0,0 +1,41 @@
+module Gitlab
+ module Sherlock
+ # Rack middleware used for tracking request metrics.
+ class Middleware
+ CONTENT_TYPES = /text\/html|application\/json/i
+
+ IGNORE_PATHS = %r{^/sherlock}
+
+ def initialize(app)
+ @app = app
+ end
+
+ # env - A Hash containing Rack environment details.
+ def call(env)
+ if instrument?(env)
+ call_with_instrumentation(env)
+ else
+ @app.call(env)
+ end
+ end
+
+ def call_with_instrumentation(env)
+ trans = transaction_from_env(env)
+ retval = trans.run { @app.call(env) }
+
+ Sherlock.collection.add(trans)
+
+ retval
+ end
+
+ def instrument?(env)
+ !!(env['HTTP_ACCEPT'] =~ CONTENT_TYPES &&
+ env['REQUEST_URI'] !~ IGNORE_PATHS)
+ end
+
+ def transaction_from_env(env)
+ Transaction.new(env['REQUEST_METHOD'], env['REQUEST_URI'])
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
new file mode 100644
index 00000000000..4917c4ae2ac
--- /dev/null
+++ b/lib/gitlab/sherlock/query.rb
@@ -0,0 +1,114 @@
+module Gitlab
+ module Sherlock
+ class Query
+ attr_reader :id, :query, :started_at, :finished_at, :backtrace
+
+ # SQL identifiers that should be prefixed with newlines.
+ PREFIX_NEWLINE = /
+ \s+(FROM
+ |(LEFT|RIGHT)?INNER\s+JOIN
+ |(LEFT|RIGHT)?OUTER\s+JOIN
+ |WHERE
+ |AND
+ |GROUP\s+BY
+ |ORDER\s+BY
+ |LIMIT
+ |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :<
+
+ # Creates a new Query using a String and a separate Array of bindings.
+ #
+ # query - A String containing a SQL query, optionally with numeric
+ # placeholders (`$1`, `$2`, etc).
+ #
+ # bindings - An Array of ActiveRecord columns and their values.
+ # started_at - The start time of the query as a Time-like object.
+ # finished_at - The completion time of the query as a Time-like object.
+ #
+ # Returns a new Query object.
+ def self.new_with_bindings(query, bindings, started_at, finished_at)
+ bindings.each_with_index do |(_, value), index|
+ quoted_value = ActiveRecord::Base.connection.quote(value)
+
+ query = query.gsub("$#{index + 1}", quoted_value)
+ end
+
+ new(query, started_at, finished_at)
+ end
+
+ # query - The SQL query as a String (without placeholders).
+ # started_at - The start time of the query as a Time-like object.
+ # finished_at - The completion time of the query as a Time-like object.
+ def initialize(query, started_at, finished_at)
+ @id = SecureRandom.uuid
+ @query = query
+ @started_at = started_at
+ @finished_at = finished_at
+ @backtrace = caller_locations.map do |loc|
+ Location.from_ruby_location(loc)
+ end
+
+ unless @query.end_with?(';')
+ @query += ';'
+ end
+ end
+
+ # Returns the query duration in milliseconds.
+ def duration
+ @duration ||= (@finished_at - @started_at) * 1000.0
+ end
+
+ def to_param
+ @id
+ end
+
+ # Returns a human readable version of the query.
+ def formatted_query
+ @formatted_query ||= format_sql(@query)
+ end
+
+ # Returns the last application frame of the backtrace.
+ def last_application_frame
+ @last_application_frame ||= @backtrace.find(&:application?)
+ end
+
+ # Returns an Array of application frames (excluding Gems and the likes).
+ def application_backtrace
+ @application_backtrace ||= @backtrace.select(&:application?)
+ end
+
+ # Returns the query plan as a String.
+ def explain
+ unless @explain
+ ActiveRecord::Base.connection.transaction do
+ @explain = raw_explain(@query).values.flatten.join("\n")
+
+ # Roll back any queries that mutate data so we don't mess up
+ # anything when running explain on an INSERT, UPDATE, DELETE, etc.
+ raise ActiveRecord::Rollback
+ end
+ end
+
+ @explain
+ end
+
+ private
+
+ def raw_explain(query)
+ if Gitlab::Database.postgresql?
+ explain = "EXPLAIN ANALYZE #{query};"
+ else
+ explain = "EXPLAIN #{query};"
+ end
+
+ ActiveRecord::Base.connection.execute(explain)
+ end
+
+ def format_sql(query)
+ query.each_line.
+ map { |line| line.strip }.
+ join("\n").
+ gsub(PREFIX_NEWLINE) { "\n#{$1} " }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb
new file mode 100644
index 00000000000..d87a4c9bb4a
--- /dev/null
+++ b/lib/gitlab/sherlock/transaction.rb
@@ -0,0 +1,131 @@
+module Gitlab
+ module Sherlock
+ class Transaction
+ attr_reader :id, :type, :path, :queries, :file_samples, :started_at,
+ :finished_at, :view_counts
+
+ # type - The type of transaction (e.g. "GET", "POST", etc)
+ # path - The path of the transaction (e.g. the HTTP request path)
+ def initialize(type, path)
+ @id = SecureRandom.uuid
+ @type = type
+ @path = path
+ @queries = []
+ @file_samples = []
+ @started_at = nil
+ @finished_at = nil
+ @thread = Thread.current
+ @view_counts = Hash.new(0)
+ end
+
+ # Runs the transaction and returns the block's return value.
+ def run
+ @started_at = Time.now
+
+ retval = with_subscriptions do
+ profile_lines { yield }
+ end
+
+ @finished_at = Time.now
+
+ retval
+ end
+
+ # Returns the duration in seconds.
+ def duration
+ @duration ||= started_at && finished_at ? finished_at - started_at : 0
+ end
+
+ def to_param
+ @id
+ end
+
+ # Returns the queries sorted in descending order by their durations.
+ def sorted_queries
+ @queries.sort { |a, b| b.duration <=> a.duration }
+ end
+
+ # Returns the file samples sorted in descending order by their durations.
+ def sorted_file_samples
+ @file_samples.sort { |a, b| b.duration <=> a.duration }
+ end
+
+ # Finds a query by the given ID.
+ #
+ # id - The query ID as a String.
+ #
+ # Returns a Query object if one could be found, nil otherwise.
+ def find_query(id)
+ @queries.find { |query| query.id == id }
+ end
+
+ # Finds a file sample by the given ID.
+ #
+ # id - The query ID as a String.
+ #
+ # Returns a FileSample object if one could be found, nil otherwise.
+ def find_file_sample(id)
+ @file_samples.find { |sample| sample.id == id }
+ end
+
+ def profile_lines
+ retval = nil
+
+ if Sherlock.enable_line_profiler?
+ retval, @file_samples = LineProfiler.new.profile { yield }
+ else
+ retval = yield
+ end
+
+ retval
+ end
+
+ def subscribe_to_active_record
+ ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, data|
+ next unless same_thread?
+
+ track_query(data[:sql].strip, data[:binds], start, finish)
+ end
+ end
+
+ def subscribe_to_action_view
+ regex = /render_(template|partial)\.action_view/
+
+ ActiveSupport::Notifications.subscribe(regex) do |_, start, finish, _, data|
+ next unless same_thread?
+
+ track_view(data[:identifier])
+ end
+ end
+
+ private
+
+ def track_query(query, bindings, start, finish)
+ @queries << Query.new_with_bindings(query, bindings, start, finish)
+ end
+
+ def track_view(path)
+ @view_counts[path] += 1
+ end
+
+ def with_subscriptions
+ ar_subscriber = subscribe_to_active_record
+ av_subscriber = subscribe_to_action_view
+
+ retval = yield
+
+ ActiveSupport::Notifications.unsubscribe(ar_subscriber)
+ ActiveSupport::Notifications.unsubscribe(av_subscriber)
+
+ retval
+ end
+
+ # In case somebody uses a multi-threaded server locally (e.g. Puma) we
+ # _only_ want to track notifications that originate from the transaction
+ # thread.
+ def same_thread?
+ Thread.current == @thread
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
new file mode 100644
index 00000000000..1cd89b3a9c4
--- /dev/null
+++ b/lib/gitlab/sql/union.rb
@@ -0,0 +1,34 @@
+module Gitlab
+ module SQL
+ # Class for building SQL UNION statements.
+ #
+ # ORDER BYs are dropped from the relations as the final sort order is not
+ # guaranteed any way.
+ #
+ # Example usage:
+ #
+ # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects)
+ # sql = union.to_sql
+ #
+ # Project.where("id IN (#{sql})")
+ class Union
+ def initialize(relations)
+ @relations = relations
+ end
+
+ def to_sql
+ # Some relations may include placeholders for prepared statements, these
+ # aren't incremented properly when joining relations together this way.
+ # By using "unprepared_statements" we remove the usage of placeholders
+ # (thus fixing this problem), at a slight performance cost.
+ fragments = ActiveRecord::Base.connection.unprepared_statement do
+ @relations.map do |rel|
+ rel.reorder(nil).to_sql
+ end
+ end
+
+ fragments.join("\nUNION\n")
+ end
+ end
+ end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index e767027dc29..0cf5292b290 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -44,7 +44,7 @@ upstream gitlab-workhorse {
## Normal HTTP host
server {
- ## Either remove "default_server" from the listen line below,
+ ## Either remove "default_server" from the listen line below,
## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab
## to be served if you visit any address that your server responds to, eg.
## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server;
@@ -67,7 +67,7 @@ server {
location / {
## Serve static files from defined root folder.
## @gitlab is a named location for the upstream fallback, see below.
- try_files $uri $uri/index.html $uri.html @gitlab;
+ try_files $uri /index.html $uri.html @gitlab;
}
## We route uploads through GitLab to prevent XSS and enforce access control.
@@ -113,6 +113,12 @@ server {
proxy_pass http://gitlab;
}
+ location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
@@ -131,7 +137,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
location @gitlab-workhorse {
+ client_max_body_size 0;
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
# gzip off;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 4d31e31f8d5..31a651c87fd 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -48,7 +48,7 @@ upstream gitlab-workhorse {
## Redirects all HTTP traffic to the HTTPS host
server {
- ## Either remove "default_server" from the listen line below,
+ ## Either remove "default_server" from the listen line below,
## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab
## to be served if you visit any address that your server responds to, eg.
## the ip address of the server (http://x.x.x.x/)
@@ -112,7 +112,7 @@ server {
location / {
## Serve static files from defined root folder.
## @gitlab is a named location for the upstream fallback, see below.
- try_files $uri $uri/index.html $uri.html @gitlab;
+ try_files $uri /index.html $uri.html @gitlab;
}
## We route uploads through GitLab to prevent XSS and enforce access control.
@@ -160,6 +160,12 @@ server {
proxy_pass http://gitlab;
}
+ location ~ ^/[\w\.-]+/[\w\.-]+/gitlab-lfs/objects {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ {
# 'Error' 418 is a hack to re-use the @gitlab-workhorse block
error_page 418 = @gitlab-workhorse;
@@ -178,7 +184,22 @@ server {
return 418;
}
+ # Build artifacts should be submitted to this location
+ location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
+ # Build artifacts should be submitted to this location
+ location ~ /ci/api/v1/builds/[0-9]+/artifacts {
+ # 'Error' 418 is a hack to re-use the @gitlab-workhorse block
+ error_page 418 = @gitlab-workhorse;
+ return 418;
+ }
+
location @gitlab-workhorse {
+ client_max_body_size 0;
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
gzip off;
diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
new file mode 100644
index 00000000000..e9587595fef
--- /dev/null
+++ b/lib/tasks/flay.rake
@@ -0,0 +1,9 @@
+desc 'Code duplication analyze via flay'
+task :flay do
+ output = %x(bundle exec flay --mass 35 app/ lib/gitlab/)
+
+ if output.include? "Similar code found"
+ puts output
+ exit 1
+ end
+end
diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
new file mode 100644
index 00000000000..3bfe999ae74
--- /dev/null
+++ b/lib/tasks/flog.rake
@@ -0,0 +1,25 @@
+desc 'Code complexity analyze via flog'
+task :flog do
+ output = %x(bundle exec flog -m app/ lib/gitlab)
+ exit_code = 0
+ minimum_score = 70
+ output = output.lines
+
+ # Skip total complexity score
+ output.shift
+
+ # Skip some trash info
+ output.shift
+
+ output.each do |line|
+ score, method = line.split(" ")
+ score = score.to_i
+
+ if score > minimum_score
+ exit_code = 1
+ puts "High complexity in #{method}. Score: #{score}"
+ end
+ end
+
+ exit exit_code
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..3c46bcea40e 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke
Rake::Task["gitlab:backup:uploads:create"].invoke
Rake::Task["gitlab:backup:builds:create"].invoke
+ Rake::Task["gitlab:backup:artifacts:create"].invoke
backup = Backup::Manager.new
backup.pack
@@ -32,6 +33,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+ Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
Rake::Task["gitlab:shell:setup"].invoke
backup.cleanup
@@ -113,6 +115,25 @@ namespace :gitlab do
end
end
+ namespace :artifacts do
+ task create: :environment do
+ $progress.puts "Dumping artifacts ... ".blue
+
+ if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+ $progress.puts "[SKIPPED]".cyan
+ else
+ Backup::Artifacts.new.dump
+ $progress.puts "done".green
+ end
+ end
+
+ task restore: :environment do
+ $progress.puts "Restoring artifacts ... ".blue
+ Backup::Artifacts.new.restore
+ $progress.puts "done".green
+ end
+ end
+
def configure_cron_mode
if ENV['CRON']
# We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake
new file mode 100644
index 00000000000..9980e0b7984
--- /dev/null
+++ b/lib/tasks/grape.rake
@@ -0,0 +1,8 @@
+namespace :grape do
+ desc 'Print compiled grape routes'
+ task routes: :environment do
+ API::API.routes.each do |route|
+ puts route
+ end
+ end
+end
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
new file mode 100644
index 00000000000..d4291f012d3
--- /dev/null
+++ b/lib/uploaded_file.rb
@@ -0,0 +1,37 @@
+require "tempfile"
+require "fileutils"
+
+# Taken from: Rack::Test::UploadedFile
+class UploadedFile
+
+ # The filename, *not* including the path, of the "uploaded" file
+ attr_reader :original_filename
+
+ # The tempfile
+ attr_reader :tempfile
+
+ # The content type of the "uploaded" file
+ attr_accessor :content_type
+
+ def initialize(path, filename, content_type = "text/plain")
+ raise "#{path} file does not exist" unless ::File.exist?(path)
+
+ @content_type = content_type
+ @original_filename = filename || ::File.basename(path)
+ @tempfile = File.new(path, 'rb')
+ end
+
+ def path
+ @tempfile.path
+ end
+
+ alias_method :local_path, :path
+
+ def method_missing(method_name, *args, &block) #:nodoc:
+ @tempfile.__send__(method_name, *args, &block)
+ end
+
+ def respond_to?(method_name, include_private = false) #:nodoc:
+ @tempfile.respond_to?(method_name, include_private) || super
+ end
+end