summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorRubén Dávila <rdavila84@gmail.com>2016-01-14 17:28:44 -0500
committerRubén Dávila <rdavila84@gmail.com>2016-01-14 17:28:44 -0500
commitc8db25c37c0d78457d06117497ccde7ad80e2321 (patch)
treedc2db4637c33c53b032a22f8e78975838396ed13 /lib
parent6b9c730e91962a6d6343bcb7fc4dc75c99b41bde (diff)
parent948bb655f3cba9909b7396c3062da7b22f4409b3 (diff)
downloadgitlab-ce-c8db25c37c0d78457d06117497ccde7ad80e2321.tar.gz
Merge branch 'master' into issue_3945
Diffstat (limited to 'lib')
-rw-r--r--lib/api/api.rb2
-rw-r--r--lib/api/builds.rb149
-rw-r--r--lib/api/entities.rb37
-rw-r--r--lib/api/helpers.rb34
-rw-r--r--lib/api/merge_requests.rb2
-rw-r--r--lib/api/notes.rb21
-rw-r--r--lib/api/projects.rb31
-rw-r--r--lib/api/tags.rb21
-rw-r--r--lib/api/triggers.rb69
-rw-r--r--lib/api/users.rb16
-rw-r--r--lib/api/variables.rb95
-rw-r--r--lib/banzai/cross_project_reference.rb2
-rw-r--r--lib/banzai/filter.rb1
-rw-r--r--lib/banzai/filter/abstract_reference_filter.rb52
-rw-r--r--lib/banzai/filter/autolink_filter.rb1
-rw-r--r--lib/banzai/filter/commit_range_reference_filter.rb2
-rw-r--r--lib/banzai/filter/commit_reference_filter.rb2
-rw-r--r--lib/banzai/filter/emoji_filter.rb1
-rw-r--r--lib/banzai/filter/external_issue_reference_filter.rb2
-rw-r--r--lib/banzai/filter/external_link_filter.rb1
-rw-r--r--lib/banzai/filter/gollum_tags_filter.rb151
-rw-r--r--lib/banzai/filter/issue_reference_filter.rb2
-rw-r--r--lib/banzai/filter/label_reference_filter.rb2
-rw-r--r--lib/banzai/filter/markdown_filter.rb1
-rw-r--r--lib/banzai/filter/merge_request_reference_filter.rb2
-rw-r--r--lib/banzai/filter/milestone_reference_filter.rb22
-rw-r--r--lib/banzai/filter/redactor_filter.rb3
-rw-r--r--lib/banzai/filter/reference_filter.rb9
-rw-r--r--lib/banzai/filter/reference_gatherer_filter.rb3
-rw-r--r--lib/banzai/filter/relative_link_filter.rb3
-rw-r--r--lib/banzai/filter/sanitization_filter.rb1
-rw-r--r--lib/banzai/filter/snippet_reference_filter.rb2
-rw-r--r--lib/banzai/filter/syntax_highlight_filter.rb1
-rw-r--r--lib/banzai/filter/table_of_contents_filter.rb1
-rw-r--r--lib/banzai/filter/task_list_filter.rb14
-rw-r--r--lib/banzai/filter/upload_link_filter.rb1
-rw-r--r--lib/banzai/filter/user_reference_filter.rb2
-rw-r--r--lib/banzai/lazy_reference.rb2
-rw-r--r--lib/banzai/pipeline.rb2
-rw-r--r--lib/banzai/pipeline/asciidoc_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/atom_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/base_pipeline.rb1
-rw-r--r--lib/banzai/pipeline/combined_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/description_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/email_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/full_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb3
-rw-r--r--lib/banzai/pipeline/note_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/plain_markdown_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/post_process_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/reference_extraction_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/single_line_pipeline.rb18
-rw-r--r--lib/banzai/pipeline/wiki_pipeline.rb11
-rw-r--r--lib/banzai/querying.rb18
-rw-r--r--lib/banzai/reference_extractor.rb2
-rw-r--r--lib/banzai/renderer.rb4
-rw-r--r--lib/ci/api/builds.rb22
-rw-r--r--lib/gitlab/backend/shell.rb12
-rw-r--r--lib/gitlab/build_data_builder.rb1
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata.rb109
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata/entry.rb119
-rw-r--r--lib/gitlab/contributions_calendar.rb4
-rw-r--r--lib/gitlab/current_settings.rb7
-rw-r--r--lib/gitlab/diff/parser.rb5
-rw-r--r--lib/gitlab/email/receiver.rb7
-rw-r--r--lib/gitlab/fogbugz_import/importer.rb4
-rw-r--r--lib/gitlab/github_import/base_formatter.rb21
-rw-r--r--lib/gitlab/github_import/comment_formatter.rb45
-rw-r--r--lib/gitlab/github_import/importer.rb83
-rw-r--r--lib/gitlab/github_import/issue_formatter.rb66
-rw-r--r--lib/gitlab/github_import/project_creator.rb3
-rw-r--r--lib/gitlab/github_import/pull_request_formatter.rb101
-rw-r--r--lib/gitlab/github_import/wiki_formatter.rb19
-rw-r--r--lib/gitlab/gitlab_import/importer.rb2
-rw-r--r--lib/gitlab/ldap/access.rb14
-rw-r--r--lib/gitlab/ldap/adapter.rb24
-rw-r--r--lib/gitlab/ldap/config.rb4
-rw-r--r--lib/gitlab/markdown/pipeline.rb2
-rw-r--r--lib/gitlab/metrics.rb87
-rw-r--r--lib/gitlab/metrics/instrumentation.rb2
-rw-r--r--lib/gitlab/metrics/metric.rb12
-rw-r--r--lib/gitlab/metrics/obfuscated_sql.rb47
-rw-r--r--lib/gitlab/metrics/rack_middleware.rb10
-rw-r--r--lib/gitlab/metrics/sampler.rb51
-rw-r--r--lib/gitlab/metrics/sidekiq_middleware.rb14
-rw-r--r--lib/gitlab/metrics/subscribers/action_view.rb11
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb30
-rw-r--r--lib/gitlab/metrics/transaction.rb61
-rw-r--r--lib/gitlab/recaptcha.rb14
-rw-r--r--lib/gitlab/reference_extractor.rb4
-rw-r--r--lib/tasks/gitlab/check.rake2
-rw-r--r--lib/tasks/gitlab/task_helpers.rake2
-rw-r--r--lib/version_check.rb2
93 files changed, 1497 insertions, 368 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 7834262d612..7efe0a0262f 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -54,5 +54,7 @@ module API
mount Keys
mount Tags
mount Triggers
+ mount Builds
+ mount Variables
end
end
diff --git a/lib/api/builds.rb b/lib/api/builds.rb
new file mode 100644
index 00000000000..d293f988165
--- /dev/null
+++ b/lib/api/builds.rb
@@ -0,0 +1,149 @@
+module API
+ # Projects builds API
+ class Builds < Grape::API
+ before { authenticate! }
+
+ resource :projects do
+ # Get a project builds
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled;
+ # if none provided showing all builds)
+ # Example Request:
+ # GET /projects/:id/builds
+ get ':id/builds' do
+ builds = user_project.builds.order('id DESC')
+ builds = filter_builds(builds, params[:scope])
+
+ present paginate(builds), with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
+
+ # Get builds for a specific commit of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # sha (required) - The SHA id of a commit
+ # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled;
+ # if none provided showing all builds)
+ # Example Request:
+ # GET /projects/:id/repository/commits/:sha/builds
+ get ':id/repository/commits/:sha/builds' do
+ commit = user_project.ci_commits.find_by_sha(params[:sha])
+ return not_found! unless commit
+
+ builds = commit.builds.order('id DESC')
+ builds = filter_builds(builds, params[:scope])
+
+ present paginate(builds), with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
+
+ # Get a specific build of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # build_id (required) - The ID of a build
+ # Example Request:
+ # GET /projects/:id/builds/:build_id
+ get ':id/builds/:build_id' do
+ build = get_build(params[:build_id])
+ return not_found!(build) unless build
+
+ present build, with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
+
+ # Get a trace of a specific build of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # build_id (required) - The ID of a build
+ # Example Request:
+ # GET /projects/:id/build/:build_id/trace
+ #
+ # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
+ # is saved in the DB instead of file). But before that, we need to consider how to replace the value of
+ # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
+ get ':id/builds/:build_id/trace' do
+ build = get_build(params[:build_id])
+ return not_found!(build) unless build
+
+ header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
+ content_type 'text/plain'
+ env['api.format'] = :binary
+
+ trace = build.trace
+ body trace
+ end
+
+ # Cancel a specific build of a project
+ #
+ # parameters:
+ # id (required) - the id of a project
+ # build_id (required) - the id of a build
+ # example request:
+ # post /projects/:id/build/:build_id/cancel
+ post ':id/builds/:build_id/cancel' do
+ authorize_manage_builds!
+
+ build = get_build(params[:build_id])
+ return not_found!(build) unless build
+
+ build.cancel
+
+ present build, with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
+
+ # Retry a specific build of a project
+ #
+ # parameters:
+ # id (required) - the id of a project
+ # build_id (required) - the id of a build
+ # example request:
+ # post /projects/:id/build/:build_id/retry
+ post ':id/builds/:build_id/retry' do
+ authorize_manage_builds!
+
+ build = get_build(params[:build_id])
+ return forbidden!('Build is not retryable') unless build && build.retryable?
+
+ build = Ci::Build.retry(build)
+
+ present build, with: Entities::Build,
+ user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
+ end
+ end
+
+ helpers do
+ def get_build(id)
+ user_project.builds.find_by(id: id.to_i)
+ end
+
+ def filter_builds(builds, scope)
+ return builds if scope.nil? || scope.empty?
+
+ available_statuses = ::CommitStatus::AVAILABLE_STATUSES
+ scope =
+ if scope.is_a?(String)
+ [scope]
+ elsif scope.is_a?(Hashie::Mash)
+ scope.values
+ else
+ ['unknown']
+ end
+
+ unknown = scope - available_statuses
+ render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
+
+ builds.where(status: available_statuses && scope)
+ end
+
+ def authorize_manage_builds!
+ authorize! :manage_builds, user_project
+ end
+ end
+ end
+end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index f8511ac5f5c..82a75734de0 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -71,6 +71,7 @@ module API
expose :avatar_url
expose :star_count, :forks_count
expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
+ expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
end
class ProjectMember < UserBasic
@@ -166,7 +167,6 @@ module API
class MergeRequest < ProjectEntity
expose :target_branch, :source_branch
- # deprecated, always returns 0
expose :upvotes, :downvotes
expose :author, :assignee, using: Entities::UserBasic
expose :source_project_id, :target_project_id
@@ -366,5 +366,40 @@ module API
class TriggerRequest < Grape::Entity
expose :id, :variables
end
+
+ class Runner < Grape::Entity
+ expose :id
+ expose :description
+ expose :active
+ expose :is_shared
+ expose :name
+ end
+
+ class Build < Grape::Entity
+ expose :id, :status, :stage, :name, :ref, :tag, :coverage
+ expose :created_at, :started_at, :finished_at
+ expose :user, with: User
+ # TODO: download_url in Ci:Build model is an GitLab Web Interface URL, not API URL. We should think on some API
+ # for downloading of artifacts (see: https://gitlab.com/gitlab-org/gitlab-ce/issues/4255)
+ expose :download_url do |repo_obj, options|
+ if options[:user_can_download_artifacts]
+ repo_obj.download_url
+ end
+ end
+ expose :commit, with: RepoCommit do |repo_obj, _options|
+ if repo_obj.respond_to?(:commit)
+ repo_obj.commit.commit_data
+ end
+ end
+ expose :runner, with: Runner
+ end
+
+ class Trigger < Grape::Entity
+ expose :token, :created_at, :updated_at, :deleted_at, :last_used
+ end
+
+ class Variable < Grape::Entity
+ expose :key, :value
+ end
end
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index a4df810e755..6d2380cf47d 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -97,11 +97,9 @@ module API
end
def paginate(relation)
- per_page = params[:per_page].to_i
- paginated = relation.page(params[:page]).per(per_page)
- add_pagination_headers(paginated, per_page)
-
- paginated
+ relation.page(params[:page]).per(params[:per_page].to_i).tap do |data|
+ add_pagination_headers(data)
+ end
end
def authenticate!
@@ -289,12 +287,14 @@ module API
# file helpers
- def uploaded_file!(field, uploads_path)
+ def uploaded_file(field, uploads_path)
if params[field]
bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
return params[field]
end
+ return nil unless params["#{field}.path"] && params["#{field}.name"]
+
# sanitize file paths
# this requires all paths to exist
required_attributes! %W(#{field}.path)
@@ -327,16 +327,26 @@ module API
private
- def add_pagination_headers(paginated, per_page)
+ def add_pagination_headers(paginated_data)
+ header 'X-Total', paginated_data.total_count.to_s
+ header 'X-Total-Pages', paginated_data.total_pages.to_s
+ header 'X-Per-Page', paginated_data.limit_value.to_s
+ header 'X-Page', paginated_data.current_page.to_s
+ header 'X-Next-Page', paginated_data.next_page.to_s
+ header 'X-Prev-Page', paginated_data.prev_page.to_s
+ header 'Link', pagination_links(paginated_data)
+ end
+
+ def pagination_links(paginated_data)
request_url = request.url.split('?').first
links = []
- links << %(<#{request_url}?page=#{paginated.current_page - 1}&per_page=#{per_page}>; rel="prev") unless paginated.first_page?
- links << %(<#{request_url}?page=#{paginated.current_page + 1}&per_page=#{per_page}>; rel="next") unless paginated.last_page?
- links << %(<#{request_url}?page=1&per_page=#{per_page}>; rel="first")
- links << %(<#{request_url}?page=#{paginated.total_pages}&per_page=#{per_page}>; rel="last")
+ links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page?
+ links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page?
+ links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first")
+ links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last")
- header 'Link', links.join(', ')
+ links.join(', ')
end
def abilities
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 3c1c6bda260..5c97fe1c88c 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -211,7 +211,7 @@ module API
unauthorized! unless merge_request.can_be_merged_by?(current_user)
not_allowed! if !merge_request.open? || merge_request.work_in_progress?
- merge_request.check_if_can_be_merged if merge_request.unchecked?
+ merge_request.check_if_can_be_merged
render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged?
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index 3efdfe2d46e..174473f5371 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -20,7 +20,19 @@ module API
# GET /projects/:id/snippets/:noteable_id/notes
get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do
@noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"])
- present paginate(@noteable.notes), with: Entities::Note
+
+ # We exclude notes that are cross-references and that cannot be viewed
+ # by the current user. By doing this exclusion at this level and not
+ # at the DB query level (which we cannot in that case), the current
+ # page can have less elements than :per_page even if
+ # there's more than one page.
+ notes =
+ # paginate() only works with a relation. This could lead to a
+ # mismatch between the pagination headers info and the actual notes
+ # array returned, but this is really a edge-case.
+ paginate(@noteable.notes).
+ reject { |n| n.cross_reference_not_visible_for?(current_user) }
+ present notes, with: Entities::Note
end
# Get a single +noteable+ note
@@ -35,7 +47,12 @@ module API
get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do
@noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"])
@note = @noteable.notes.find(params[:note_id])
- present @note, with: Entities::Note
+
+ if @note.cross_reference_not_visible_for?(current_user)
+ not_found!("Note")
+ else
+ present @note, with: Entities::Note
+ end
end
# Create a new +noteable+ note
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index a9e0960872a..71bb342f844 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -3,7 +3,7 @@ module API
class Projects < Grape::API
before { authenticate! }
- resource :projects do
+ resource :projects, requirements: { id: /[^\/]+/ } do
helpers do
def map_public_to_visibility_level(attrs)
publik = attrs.delete(:public)
@@ -69,7 +69,8 @@ module API
# Example Request:
# GET /projects/:id
get ":id" do
- present user_project, with: Entities::ProjectWithAccess, user: current_user
+ present user_project, with: Entities::ProjectWithAccess, user: current_user,
+ user_can_admin_project: can?(current_user, :admin_project, user_project)
end
# Get events for a single project
@@ -118,7 +119,8 @@ module API
attrs = map_public_to_visibility_level(attrs)
@project = ::Projects::CreateService.new(current_user, attrs).execute
if @project.saved?
- present @project, with: Entities::Project
+ present @project, with: Entities::Project,
+ user_can_admin_project: can?(current_user, :admin_project, @project)
else
if @project.errors[:limit_reached].present?
error!(@project.errors[:limit_reached], 403)
@@ -163,7 +165,8 @@ module API
attrs = map_public_to_visibility_level(attrs)
@project = ::Projects::CreateService.new(user, attrs).execute
if @project.saved?
- present @project, with: Entities::Project
+ present @project, with: Entities::Project,
+ user_can_admin_project: can?(current_user, :admin_project, @project)
else
render_validation_error!(@project)
end
@@ -182,8 +185,9 @@ module API
if @forked_project.errors.any?
conflict!(@forked_project.errors.messages)
else
- present @forked_project, with: Entities::Project
- end
+ present @forked_project, with: Entities::Project,
+ user_can_admin_project: can?(current_user, :admin_project, @forked_project)
+ end
end
# Update an existing project
@@ -229,7 +233,8 @@ module API
if user_project.errors.any?
render_validation_error!(user_project)
else
- present user_project, with: Entities::Project
+ present user_project, with: Entities::Project,
+ user_can_admin_project: can?(current_user, :admin_project, user_project)
end
end
@@ -269,7 +274,7 @@ module API
# Remove a forked_from relationship
#
# Parameters:
- # id: (required) - The ID of the project being marked as a fork
+ # id: (required) - The ID of the project being marked as a fork
# Example Request:
# DELETE /projects/:id/fork
delete ":id/fork" do
@@ -278,6 +283,16 @@ module API
user_project.forked_project_link.destroy
end
end
+
+ # Upload a file
+ #
+ # Parameters:
+ # id: (required) - The ID of the project
+ # file: (required) - The file to be uploaded
+ post ":id/uploads" do
+ ::Projects::UploadService.new(user_project, params[:file]).execute
+ end
+
# search for projects current_user has access to
#
# Parameters:
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 47621f443e6..2d8a9e51bb9 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -40,6 +40,27 @@ module API
end
end
+ # Delete tag
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # tag_name (required) - The name of the tag
+ # Example Request:
+ # DELETE /projects/:id/repository/tags/:tag
+ delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do
+ authorize_push_project
+ result = DeleteTagService.new(user_project, current_user).
+ execute(params[:tag_name])
+
+ if result[:status] == :success
+ {
+ tag_name: params[:tag_name]
+ }
+ else
+ render_api_error!(result[:message], result[:return_code])
+ end
+ end
+
# Add release notes to tag
#
# Parameters:
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index 2781f1cf191..5e4964f446c 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -43,6 +43,75 @@ module API
render_api_error!(errors, 400)
end
end
+
+ # Get triggers list
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # page (optional) - The page number for pagination
+ # per_page (optional) - The value of items per page to show
+ # Example Request:
+ # GET /projects/:id/triggers
+ get ':id/triggers' do
+ authenticate!
+ authorize_admin_project
+
+ triggers = user_project.triggers.includes(:trigger_requests)
+ triggers = paginate(triggers)
+
+ present triggers, with: Entities::Trigger
+ end
+
+ # Get specific trigger of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # token (required) - The `token` of a trigger
+ # Example Request:
+ # GET /projects/:id/triggers/:token
+ get ':id/triggers/:token' do
+ authenticate!
+ authorize_admin_project
+
+ trigger = user_project.triggers.find_by(token: params[:token].to_s)
+ return not_found!('Trigger') unless trigger
+
+ present trigger, with: Entities::Trigger
+ end
+
+ # Create trigger
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # Example Request:
+ # POST /projects/:id/triggers
+ post ':id/triggers' do
+ authenticate!
+ authorize_admin_project
+
+ trigger = user_project.triggers.create
+
+ present trigger, with: Entities::Trigger
+ end
+
+ # Delete trigger
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # token (required) - The `token` of a trigger
+ # Example Request:
+ # DELETE /projects/:id/triggers/:token
+ delete ':id/triggers/:token' do
+ authenticate!
+ authorize_admin_project
+
+ trigger = user_project.triggers.find_by(token: params[:token].to_s)
+ return not_found!('Trigger') unless trigger
+
+ trigger.destroy
+
+ present trigger, with: Entities::Trigger
+ end
end
end
end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 3400f0713ef..fd2128bd179 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -39,7 +39,7 @@ module API
if current_user.is_admin?
present @user, with: Entities::UserFull
else
- present @user, with: Entities::UserBasic
+ present @user, with: Entities::User
end
end
@@ -284,10 +284,12 @@ module API
authenticated_as_admin!
user = User.find_by(id: params[:id])
- if user
+ if !user
+ not_found!('User')
+ elsif !user.ldap_blocked?
user.block
else
- not_found!('User')
+ forbidden!('LDAP blocked users cannot be modified by the API')
end
end
@@ -299,10 +301,12 @@ module API
authenticated_as_admin!
user = User.find_by(id: params[:id])
- if user
- user.activate
- else
+ if !user
not_found!('User')
+ elsif user.ldap_blocked?
+ forbidden!('LDAP blocked users cannot be unblocked by the API')
+ else
+ user.activate
end
end
end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
new file mode 100644
index 00000000000..d9a055f6c92
--- /dev/null
+++ b/lib/api/variables.rb
@@ -0,0 +1,95 @@
+module API
+ # Projects variables API
+ class Variables < Grape::API
+ before { authenticate! }
+ before { authorize_admin_project }
+
+ resource :projects do
+ # Get project variables
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # page (optional) - The page number for pagination
+ # per_page (optional) - The value of items per page to show
+ # Example Request:
+ # GET /projects/:id/variables
+ get ':id/variables' do
+ variables = user_project.variables
+ present paginate(variables), with: Entities::Variable
+ end
+
+ # Get specific variable of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # key (required) - The `key` of variable
+ # Example Request:
+ # GET /projects/:id/variables/:key
+ get ':id/variables/:key' do
+ key = params[:key]
+ variable = user_project.variables.find_by(key: key.to_s)
+
+ return not_found!('Variable') unless variable
+
+ present variable, with: Entities::Variable
+ end
+
+ # Create a new variable in project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # key (required) - The key of variable
+ # value (required) - The value of variable
+ # Example Request:
+ # POST /projects/:id/variables
+ post ':id/variables' do
+ required_attributes! [:key, :value]
+
+ variable = user_project.variables.create(key: params[:key], value: params[:value])
+
+ if variable.valid?
+ present variable, with: Entities::Variable
+ else
+ render_validation_error!(variable)
+ end
+ end
+
+ # Update existing variable of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # key (optional) - The `key` of variable
+ # value (optional) - New value for `value` field of variable
+ # Example Request:
+ # PUT /projects/:id/variables/:key
+ put ':id/variables/:key' do
+ variable = user_project.variables.find_by(key: params[:key].to_s)
+
+ return not_found!('Variable') unless variable
+
+ attrs = attributes_for_keys [:value]
+ if variable.update(attrs)
+ present variable, with: Entities::Variable
+ else
+ render_validation_error!(variable)
+ end
+ end
+
+ # Delete existing variable of a project
+ #
+ # Parameters:
+ # id (required) - The ID of a project
+ # key (required) - The ID of a variable
+ # Example Request:
+ # DELETE /projects/:id/variables/:key
+ delete ':id/variables/:key' do
+ variable = user_project.variables.find_by(key: params[:key].to_s)
+
+ return not_found!('Variable') unless variable
+ variable.destroy
+
+ present variable, with: Entities::Variable
+ end
+ end
+ end
+end
diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb
index ba2866e1efa..0257848b6bc 100644
--- a/lib/banzai/cross_project_reference.rb
+++ b/lib/banzai/cross_project_reference.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
# Common methods for ReferenceFilters that support an optional cross-project
# reference.
diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb
index fd4fe024252..905c4c0144e 100644
--- a/lib/banzai/filter.rb
+++ b/lib/banzai/filter.rb
@@ -1,5 +1,4 @@
require 'active_support/core_ext/string/output_safety'
-require 'banzai'
module Banzai
module Filter
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 63ad8910c0f..cdbaecf8d90 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# Issues, Merge Requests, Snippets, Commits and Commit Ranges share
@@ -47,7 +45,17 @@ module Banzai
{ object_sym => LazyReference.new(object_class, node.attr(data_reference)) }
end
- delegate :object_class, :object_sym, :references_in, to: :class
+ def object_class
+ self.class.object_class
+ end
+
+ def object_sym
+ self.class.object_sym
+ end
+
+ def references_in(*args, &block)
+ self.class.references_in(*args, &block)
+ end
def find_object(project, id)
# Implement in child class
@@ -60,27 +68,31 @@ module Banzai
end
def call
- # `#123`
- replace_text_nodes_matching(object_class.reference_pattern) do |content|
- object_link_filter(content, object_class.reference_pattern)
- end
+ if object_class.reference_pattern
+ # `#123`
+ replace_text_nodes_matching(object_class.reference_pattern) do |content|
+ object_link_filter(content, object_class.reference_pattern)
+ end
- # `[Issue](#123)`, which is turned into
- # `<a href="#123">Issue</a>`
- replace_link_nodes_with_href(object_class.reference_pattern) do |link, text|
- object_link_filter(link, object_class.reference_pattern, link_text: text)
+ # `[Issue](#123)`, which is turned into
+ # `<a href="#123">Issue</a>`
+ replace_link_nodes_with_href(object_class.reference_pattern) do |link, text|
+ object_link_filter(link, object_class.reference_pattern, link_text: text)
+ end
end
- # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into
- # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>`
- replace_link_nodes_with_text(object_class.link_reference_pattern) do |text|
- object_link_filter(text, object_class.link_reference_pattern)
- end
+ if object_class.link_reference_pattern
+ # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into
+ # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>`
+ replace_link_nodes_with_text(object_class.link_reference_pattern) do |text|
+ object_link_filter(text, object_class.link_reference_pattern)
+ end
- # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into
- # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>`
- replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text|
- object_link_filter(link, object_class.link_reference_pattern, link_text: text)
+ # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into
+ # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>`
+ replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text|
+ object_link_filter(link, object_class.link_reference_pattern, link_text: text)
+ end
end
end
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index da4ee80c1b5..856f56fb175 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
require 'uri'
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index e67cd45ab9b..470727ee312 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces commit range references with links.
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
index 9e57608b483..713a56ba949 100644
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ b/lib/banzai/filter/commit_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces commit references with links.
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index 86838e1483c..5952a031626 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -1,5 +1,4 @@
require 'action_controller'
-require 'banzai'
require 'gitlab_emoji'
require 'html/pipeline/filter'
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index 6136e73c096..edc26386903 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces external issue tracker references with links.
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index ac87b9820af..8d368f3b9e7 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
new file mode 100644
index 00000000000..fe01dae4850
--- /dev/null
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -0,0 +1,151 @@
+require 'banzai'
+require 'html/pipeline/filter'
+
+module Banzai
+ module Filter
+ # HTML Filter for parsing Gollum's tags in HTML. It's only parses the
+ # following tags:
+ #
+ # - Link to internal pages:
+ #
+ # * [[Bug Reports]]
+ # * [[How to Contribute|Contributing]]
+ #
+ # - Link to external resources:
+ #
+ # * [[http://en.wikipedia.org/wiki/Git_(software)]]
+ # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]]
+ #
+ # - Link internal images, the special attributes will be ignored:
+ #
+ # * [[images/logo.png]]
+ # * [[images/logo.png|alt=Logo]]
+ #
+ # - Link external images, the special attributes will be ignored:
+ #
+ # * [[http://example.com/images/logo.png]]
+ # * [[http://example.com/images/logo.png|alt=Logo]]
+ #
+ # Based on Gollum::Filter::Tags
+ #
+ # Context options:
+ # :project_wiki (required) - Current project wiki.
+ #
+ class GollumTagsFilter < HTML::Pipeline::Filter
+ include ActionView::Helpers::TagHelper
+
+ # Pattern to match tags content that should be parsed in HTML.
+ #
+ # Gollum's tags have been made to resemble the tags of other markups,
+ # especially MediaWiki. The basic syntax is:
+ #
+ # [[tag]]
+ #
+ # Some tags will accept attributes which are separated by pipe
+ # symbols.Some attributes must precede the tag and some must follow it:
+ #
+ # [[prefix-attribute|tag]]
+ # [[tag|suffix-attribute]]
+ #
+ # See https://github.com/gollum/gollum/wiki
+ #
+ # Rubular: http://rubular.com/r/7dQnE5CUCH
+ TAGS_PATTERN = %r{\[\[(.+?)\]\]}
+
+ # Pattern to match allowed image extensions
+ ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i
+
+ def call
+ search_text_nodes(doc).each do |node|
+ content = node.content
+
+ next unless content.match(TAGS_PATTERN)
+
+ html = process_tag($1)
+
+ if html && html != node.content
+ node.replace(html)
+ end
+ end
+
+ doc
+ end
+
+ private
+
+ # Process a single tag into its final HTML form.
+ #
+ # tag - The String tag contents (the stuff inside the double brackets).
+ #
+ # Returns the String HTML version of the tag.
+ def process_tag(tag)
+ parts = tag.split('|')
+
+ return if parts.size.zero?
+
+ process_image_tag(parts) || process_page_link_tag(parts)
+ end
+
+ # Attempt to process the tag as an image tag.
+ #
+ # tag - The String tag contents (the stuff inside the double brackets).
+ #
+ # Returns the String HTML if the tag is a valid image tag or nil
+ # if it is not.
+ def process_image_tag(parts)
+ content = parts[0].strip
+
+ return unless image?(content)
+
+ if url?(content)
+ path = content
+ elsif file = project_wiki.find_file(content)
+ path = ::File.join project_wiki_base_path, file.path
+ end
+
+ if path
+ content_tag(:img, nil, src: path)
+ end
+ end
+
+ def image?(path)
+ path =~ ALLOWED_IMAGE_EXTENSIONS
+ end
+
+ def url?(path)
+ path.start_with?(*%w(http https))
+ end
+
+ # Attempt to process the tag as a page link tag.
+ #
+ # tag - The String tag contents (the stuff inside the double brackets).
+ #
+ # Returns the String HTML if the tag is a valid page link tag or nil
+ # if it is not.
+ def process_page_link_tag(parts)
+ if parts.size == 1
+ url = parts[0].strip
+ else
+ name, url = *parts.compact.map(&:strip)
+ end
+
+ content_tag(:a, name || url, href: url)
+ end
+
+ def project_wiki
+ context[:project_wiki]
+ end
+
+ def project_wiki_base_path
+ project_wiki && project_wiki.wiki_base_path
+ end
+
+ # Ensure that a :project_wiki key exists in context
+ #
+ # Note that while the key might exist, its value could be nil!
+ def validate
+ needs :project_wiki
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 51180cb901a..9f08aa36e8b 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces issue references with links. References to
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index a3a7a23c1e6..95e7d209119 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces label references with links.
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index d09cf41df39..0659fed1419 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index 755b946a34b..57c71708992 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces merge request references with links. References
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
new file mode 100644
index 00000000000..e88b27c1fae
--- /dev/null
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -0,0 +1,22 @@
+require 'banzai'
+
+module Banzai
+ module Filter
+ # HTML filter that replaces milestone references with links.
+ class MilestoneReferenceFilter < AbstractReferenceFilter
+ def self.object_class
+ Milestone
+ end
+
+ def find_object(project, id)
+ project.milestones.find_by(iid: id)
+ end
+
+ def url_for_object(issue, project)
+ h = Gitlab::Application.routes.url_helpers
+ h.namespace_project_milestone_url(project.namespace, project, milestone,
+ only_path: context[:only_path])
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index f01a32b5ae5..7141ed7c9bd 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
@@ -10,7 +9,7 @@ module Banzai
#
class RedactorFilter < HTML::Pipeline::Filter
def call
- doc.css('a.gfm').each do |node|
+ Querying.css(doc, 'a.gfm').each do |node|
unless user_can_see_reference?(node)
# The reference should be replaced by the original text,
# which is not always the same as the rendered text.
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 8ca05ace88c..20bd4f7ee6e 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -1,5 +1,4 @@
require 'active_support/core_ext/string/output_safety'
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
@@ -124,7 +123,7 @@ module Banzai
def replace_link_nodes_with_text(pattern)
return doc if project.nil?
- doc.search('a').each do |node|
+ doc.xpath('descendant-or-self::a').each do |node|
klass = node.attr('class')
next if klass && klass.include?('gfm')
@@ -133,7 +132,7 @@ module Banzai
next unless link && text
- link = URI.decode(link)
+ link = CGI.unescape(link)
# Ignore ending punctionation like periods or commas
next unless link == text && text =~ /\A#{pattern}/
@@ -162,7 +161,7 @@ module Banzai
def replace_link_nodes_with_href(pattern)
return doc if project.nil?
- doc.search('a').each do |node|
+ doc.xpath('descendant-or-self::a').each do |node|
klass = node.attr('class')
next if klass && klass.include?('gfm')
@@ -170,7 +169,7 @@ module Banzai
text = node.text
next unless link && text
- link = URI.decode(link)
+ link = CGI.unescape(link)
next unless link && link =~ /\A#{pattern}\z/
html = yield link, text
diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb
index 12412ff7ea9..86d484feb90 100644
--- a/lib/banzai/filter/reference_gatherer_filter.rb
+++ b/lib/banzai/filter/reference_gatherer_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
@@ -16,7 +15,7 @@ module Banzai
end
def call
- doc.css('a.gfm').each do |node|
+ Querying.css(doc, 'a.gfm').each do |node|
gather_references(node)
end
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 5a081125f21..41380627d39 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
require 'uri'
@@ -91,7 +90,7 @@ module Banzai
parts = request_path.split('/')
parts.pop if path_type(request_path) != 'tree'
- while parts.length > 1 && path.start_with?('../')
+ while path.start_with?('../')
parts.pop
path.sub!('../', '')
end
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index d03e3ae4b3c..3f49d492f2f 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
require 'html/pipeline/sanitization_filter'
diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb
index 1ad5df96f85..c870a42f741 100644
--- a/lib/banzai/filter/snippet_reference_filter.rb
+++ b/lib/banzai/filter/snippet_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces snippet references with links. References to
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index c889cc1e97c..8c5855e5ffc 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
require 'rouge/plugins/redcarpet'
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index 9b3e67206d5..4056dcd6d64 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
module Banzai
diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb
index bdf7c2ebdfc..66608c9859c 100644
--- a/lib/banzai/filter/task_list_filter.rb
+++ b/lib/banzai/filter/task_list_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'task_list/filter'
module Banzai
@@ -12,13 +11,18 @@ module Banzai
#
# See https://github.com/github/task_list/pull/60
class TaskListFilter < TaskList::Filter
- def add_css_class(node, *new_class_names)
+ def add_css_class_with_fix(node, *new_class_names)
if new_class_names.include?('task-list')
- super if node.children.any? { |c| c['class'] == 'task-list-item' }
- else
- super
+ # Don't add class to all lists
+ return
+ elsif new_class_names.include?('task-list-item')
+ add_css_class_without_fix(node.parent, 'task-list')
end
+
+ add_css_class_without_fix(node, *new_class_names)
end
+
+ alias_method_chain :add_css_class, :fix
end
end
end
diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb
index 1a1d0aad8ca..f642aee0967 100644
--- a/lib/banzai/filter/upload_link_filter.rb
+++ b/lib/banzai/filter/upload_link_filter.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline/filter'
require 'uri'
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index 964ab60f614..24f16f8b547 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Filter
# HTML filter that replaces user or group references with links.
diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb
index 073ec5d9801..1095b4debc7 100644
--- a/lib/banzai/lazy_reference.rb
+++ b/lib/banzai/lazy_reference.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
class LazyReference
def self.load(refs)
diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb
index 4e017809d9d..142a9962eb1 100644
--- a/lib/banzai/pipeline.rb
+++ b/lib/banzai/pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
def self.[](name)
diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb
index 5e76a817be5..f1331c0ebf9 100644
--- a/lib/banzai/pipeline/asciidoc_pipeline.rb
+++ b/lib/banzai/pipeline/asciidoc_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class AsciidocPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb
index 957f352aec5..9694e4bc23f 100644
--- a/lib/banzai/pipeline/atom_pipeline.rb
+++ b/lib/banzai/pipeline/atom_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class AtomPipeline < FullPipeline
diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb
index cd30009e5c0..db5177db7b3 100644
--- a/lib/banzai/pipeline/base_pipeline.rb
+++ b/lib/banzai/pipeline/base_pipeline.rb
@@ -1,4 +1,3 @@
-require 'banzai'
require 'html/pipeline'
module Banzai
diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb
index f3bf1809d18..9485199132e 100644
--- a/lib/banzai/pipeline/combined_pipeline.rb
+++ b/lib/banzai/pipeline/combined_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
module CombinedPipeline
diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb
index 94c2cb165a5..20e24ace352 100644
--- a/lib/banzai/pipeline/description_pipeline.rb
+++ b/lib/banzai/pipeline/description_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class DescriptionPipeline < FullPipeline
diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb
index 14356145a35..e47c384afc1 100644
--- a/lib/banzai/pipeline/email_pipeline.rb
+++ b/lib/banzai/pipeline/email_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class EmailPipeline < FullPipeline
diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb
index 72395a5d50e..d47ddfda4be 100644
--- a/lib/banzai/pipeline/full_pipeline.rb
+++ b/lib/banzai/pipeline/full_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 38750b55ec7..b7a38ea8427 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class GfmPipeline < BasePipeline
@@ -22,6 +20,7 @@ module Banzai
Filter::CommitRangeReferenceFilter,
Filter::CommitReferenceFilter,
Filter::LabelReferenceFilter,
+ Filter::MilestoneReferenceFilter,
Filter::TaskListFilter
]
diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb
index 89335143852..7890f20f716 100644
--- a/lib/banzai/pipeline/note_pipeline.rb
+++ b/lib/banzai/pipeline/note_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class NotePipeline < FullPipeline
diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb
index 998fd75daa2..3fbc681457b 100644
--- a/lib/banzai/pipeline/plain_markdown_pipeline.rb
+++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class PlainMarkdownPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb
index 148f24b6ce1..bd338c045f3 100644
--- a/lib/banzai/pipeline/post_process_pipeline.rb
+++ b/lib/banzai/pipeline/post_process_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class PostProcessPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb
index 4f9bc9fcccc..eaddccd30a5 100644
--- a/lib/banzai/pipeline/reference_extraction_pipeline.rb
+++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
module Pipeline
class ReferenceExtractionPipeline < BasePipeline
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index 6725c9039a9..8b84ab401df 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -1,9 +1,23 @@
-require 'banzai'
-
module Banzai
module Pipeline
class SingleLinePipeline < GfmPipeline
+ def self.filters
+ @filters ||= [
+ Filter::SanitizationFilter,
+
+ Filter::EmojiFilter,
+ Filter::AutolinkFilter,
+ Filter::ExternalLinkFilter,
+ Filter::UserReferenceFilter,
+ Filter::IssueReferenceFilter,
+ Filter::ExternalIssueReferenceFilter,
+ Filter::MergeRequestReferenceFilter,
+ Filter::SnippetReferenceFilter,
+ Filter::CommitRangeReferenceFilter,
+ Filter::CommitReferenceFilter,
+ ]
+ end
end
end
end
diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb
new file mode 100644
index 00000000000..50b5450e70b
--- /dev/null
+++ b/lib/banzai/pipeline/wiki_pipeline.rb
@@ -0,0 +1,11 @@
+require 'banzai'
+
+module Banzai
+ module Pipeline
+ class WikiPipeline < FullPipeline
+ def self.filters
+ super.insert(1, Filter::GollumTagsFilter)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb
new file mode 100644
index 00000000000..1e1b51e683e
--- /dev/null
+++ b/lib/banzai/querying.rb
@@ -0,0 +1,18 @@
+module Banzai
+ module Querying
+ # Searches a Nokogiri document using a CSS query, optionally optimizing it
+ # whenever possible.
+ #
+ # document - A document/element to search.
+ # query - The CSS query to use.
+ #
+ # Returns a Nokogiri::XML::NodeSet.
+ def self.css(document, query)
+ # When using "a.foo" Nokogiri compiles this to "//a[...]" but
+ # "descendant::a[...]" is quite a bit faster and achieves the same result.
+ xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::')
+
+ document.xpath(xpath)
+ end
+ end
+end
diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb
index 2c197d31898..f4079538ec5 100644
--- a/lib/banzai/reference_extractor.rb
+++ b/lib/banzai/reference_extractor.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Banzai
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 115ae914524..891c0fd7749 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,7 +1,5 @@
module Banzai
module Renderer
- CACHE_ENABLED = false
-
# Convert a Markdown String into an HTML-safe String of HTML
#
# Note that while the returned HTML will have been sanitized of dangerous
@@ -20,7 +18,7 @@ module Banzai
cache_key = context.delete(:cache_key)
cache_key = full_cache_key(cache_key, context[:pipeline])
- if cache_key && CACHE_ENABLED
+ if cache_key
Rails.cache.fetch(cache_key) do
cacheless_render(text, context)
end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 15faa6edd84..fb87637b94f 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -78,11 +78,13 @@ module Ci
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
- # file (required) - The uploaded file
+ # file (required) - Artifacts file
# Parameters (accelerated by GitLab Workhorse):
# file.path - path to locally stored body (generated by Workhorse)
# file.name - real filename as send in Content-Disposition
# file.type - real content type as send in Content-Type
+ # metadata.path - path to locally stored body (generated by Workhorse)
+ # metadata.name - filename (generated by Workhorse)
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Body:
@@ -96,13 +98,20 @@ module Ci
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
- forbidden!('build is not running') unless build.running?
+ forbidden!('Build is not running!') unless build.running?
- file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
- file_to_large! unless file.size < max_artifacts_size
+ artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+ artifacts = uploaded_file(:file, artifacts_upload_path)
+ metadata = uploaded_file(:metadata, artifacts_upload_path)
- if build.update_attributes(artifacts_file: file)
- present build, with: Entities::Build
+ bad_request!('Missing artifacts file!') unless artifacts
+ file_to_large! unless artifacts.size < max_artifacts_size
+
+ build.artifacts_file = artifacts
+ build.artifacts_metadata = metadata
+
+ if build.save
+ present(build, with: Entities::Build)
else
render_validation_error!(build)
end
@@ -148,6 +157,7 @@ module Ci
not_found! unless build
authenticate_build_token!(build)
build.remove_artifacts_file!
+ build.remove_artifacts_metadata!
end
end
end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 459e3d6bcdb..4c15d58d680 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -150,6 +150,18 @@ module Gitlab
"#{path}.git", tag_name])
end
+ # Gc repository
+ #
+ # path - project path with namespace
+ #
+ # Ex.
+ # gc("gitlab/gitlab-ci")
+ #
+ def gc(path)
+ Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
+ "#{path}.git"])
+ end
+
# Add new key to gitlab-shell
#
# Ex.
diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb
index 86bfa0a4378..34e949130da 100644
--- a/lib/gitlab/build_data_builder.rb
+++ b/lib/gitlab/build_data_builder.rb
@@ -23,6 +23,7 @@ module Gitlab
build_started_at: build.started_at,
build_finished_at: build.finished_at,
build_duration: build.duration,
+ build_allow_failure: build.allow_failure,
# TODO: do we still need it?
project_id: project.id,
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
new file mode 100644
index 00000000000..1344f5d120b
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -0,0 +1,109 @@
+require 'zlib'
+require 'json'
+
+module Gitlab
+ module Ci
+ module Build
+ module Artifacts
+ class Metadata
+ class ParserError < StandardError; end
+
+ VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
+ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
+
+ attr_reader :file, :path, :full_version
+
+ def initialize(file, path)
+ @file, @path = file, path
+ @full_version = read_version
+ end
+
+ def version
+ @full_version.match(VERSION_PATTERN)[1]
+ end
+
+ def errors
+ gzip do |gz|
+ read_string(gz) # version
+ errors = read_string(gz)
+ raise ParserError, 'Errors field not found!' unless errors
+
+ begin
+ JSON.parse(errors)
+ rescue JSON::ParserError
+ raise ParserError, 'Invalid errors field!'
+ end
+ end
+ end
+
+ def find_entries!
+ gzip do |gz|
+ 2.times { read_string(gz) } # version and errors fields
+ match_entries(gz)
+ end
+ end
+
+ def to_entry
+ entries = find_entries!
+ Entry.new(@path, entries)
+ end
+
+ private
+
+ def match_entries(gz)
+ entries = {}
+ match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$}
+
+ until gz.eof? do
+ begin
+ path = read_string(gz).force_encoding('UTF-8')
+ meta = read_string(gz).force_encoding('UTF-8')
+
+ next unless path.valid_encoding? && meta.valid_encoding?
+ next unless path =~ match_pattern
+ next if path =~ INVALID_PATH_PATTERN
+
+ entries[path] = JSON.parse(meta, symbolize_names: true)
+ rescue JSON::ParserError, Encoding::CompatibilityError
+ next
+ end
+ end
+
+ entries
+ end
+
+ def read_version
+ gzip do |gz|
+ version_string = read_string(gz)
+
+ unless version_string
+ raise ParserError, 'Artifacts metadata file empty!'
+ end
+
+ unless version_string =~ VERSION_PATTERN
+ raise ParserError, 'Invalid version!'
+ end
+
+ version_string.chomp
+ end
+ end
+
+ def read_uint32(gz)
+ binary = gz.read(4)
+ binary.unpack('L>')[0] if binary
+ end
+
+ def read_string(gz)
+ string_size = read_uint32(gz)
+ return nil unless string_size
+ gz.read(string_size)
+ end
+
+ def gzip(&block)
+ Zlib::GzipReader.open(@file, &block)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
new file mode 100644
index 00000000000..25b71fc3275
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -0,0 +1,119 @@
+module Gitlab
+ module Ci::Build::Artifacts
+ class Metadata
+ ##
+ # Class that represents an entry (path and metadata) to a file or
+ # directory in GitLab CI Build Artifacts binary file / archive
+ #
+ # This is IO-operations safe class, that does similar job to
+ # Ruby's Pathname but without the risk of accessing filesystem.
+ #
+ # This class is working only with UTF-8 encoded paths.
+ #
+ class Entry
+ attr_reader :path, :entries
+ attr_accessor :name
+
+ def initialize(path, entries)
+ @path = path.dup.force_encoding('UTF-8')
+ @entries = entries
+
+ if path.include?("\0")
+ raise ArgumentError, 'Path contains zero byte character!'
+ end
+
+ unless path.valid_encoding?
+ raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+ end
+ end
+
+ def directory?
+ blank_node? || @path.end_with?('/')
+ end
+
+ def file?
+ !directory?
+ end
+
+ def has_parent?
+ nodes > 0
+ end
+
+ def parent
+ return nil unless has_parent?
+ self.class.new(@path.chomp(basename), @entries)
+ end
+
+ def basename
+ (directory? && !blank_node?) ? name + '/' : name
+ end
+
+ def name
+ @name || @path.split('/').last.to_s
+ end
+
+ def children
+ return [] unless directory?
+ return @children if @children
+
+ child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
+ @children = select_entries { |path| path =~ child_pattern }
+ end
+
+ def directories(opts = {})
+ return [] unless directory?
+ dirs = children.select(&:directory?)
+ return dirs unless has_parent? && opts[:parent]
+
+ dotted_parent = parent
+ dotted_parent.name = '..'
+ dirs.prepend(dotted_parent)
+ end
+
+ def files
+ return [] unless directory?
+ children.select(&:file?)
+ end
+
+ def metadata
+ @entries[@path] || {}
+ end
+
+ def nodes
+ @path.count('/') + (file? ? 1 : 0)
+ end
+
+ def blank_node?
+ @path.empty? # "" is considered to be './'
+ end
+
+ def exists?
+ blank_node? || @entries.include?(@path)
+ end
+
+ def empty?
+ children.empty?
+ end
+
+ def to_s
+ @path
+ end
+
+ def ==(other)
+ @path == other.path && @entries == other.entries
+ end
+
+ def inspect
+ "#{self.class.name}: #{@path}"
+ end
+
+ private
+
+ def select_entries
+ selected = @entries.select { |path, _metadata| yield path }
+ selected.map { |path, _metadata| self.class.new(path, @entries) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 8a7f8dc5003..85583dce9ee 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -45,11 +45,11 @@ module Gitlab
end
def starting_year
- (Time.now - 1.year).strftime("%Y")
+ 1.year.ago.year
end
def starting_month
- Date.today.strftime("%m").to_i
+ Date.today.month
end
end
end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 46a4ef0e31f..7f938780ab1 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -38,7 +38,12 @@ module Gitlab
true
end
- use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings')
+ use_db && ActiveRecord::Base.connection.active? &&
+ !ActiveRecord::Migrator.needs_migration? &&
+ ActiveRecord::Base.connection.table_exists?('application_settings')
+
+ rescue ActiveRecord::NoDatabaseError
+ false
end
end
end
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
index 177bad4b1cf..2c0a866e8ba 100644
--- a/lib/gitlab/diff/parser.rb
+++ b/lib/gitlab/diff/parser.rb
@@ -53,8 +53,9 @@ module Gitlab
private
def filename?(line)
- line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
- '--- /tmp/diffy', '+++ /tmp/diffy')
+ line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
+ '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit
+ '--- /tmp/diffy', '+++ /tmp/diffy')
end
def identification_type(line)
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 2b252b32887..2ca21af5bc8 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -74,7 +74,7 @@ module Gitlab
def sent_notification
return nil unless reply_key
-
+
SentNotification.for(reply_key)
end
@@ -82,10 +82,7 @@ module Gitlab
attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project)
attachments.each do |link|
- text = "[#{link[:alt]}](#{link[:url]})"
- text.prepend("!") if link[:is_image]
-
- reply << "\n\n#{text}"
+ reply << "\n\n#{link[:markdown]}"
end
reply
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index 403ebeec474..db580b5e578 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -232,9 +232,7 @@ module Gitlab
return nil if res.nil?
- text = "[#{res['alt']}](#{res['url']})"
- text = "!#{text}" if res['is_image']
- text
+ res[:markdown]
end
def build_attachment_url(rel_url)
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
new file mode 100644
index 00000000000..202263c6742
--- /dev/null
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -0,0 +1,21 @@
+module Gitlab
+ module GithubImport
+ class BaseFormatter
+ attr_reader :formatter, :project, :raw_data
+
+ def initialize(project, raw_data)
+ @project = project
+ @raw_data = raw_data
+ @formatter = Gitlab::ImportFormatter.new
+ end
+
+ private
+
+ def gl_user_id(github_id)
+ User.joins(:identities).
+ find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s).
+ try(:id)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb
new file mode 100644
index 00000000000..7d58e53991a
--- /dev/null
+++ b/lib/gitlab/github_import/comment_formatter.rb
@@ -0,0 +1,45 @@
+module Gitlab
+ module GithubImport
+ class CommentFormatter < BaseFormatter
+ def attributes
+ {
+ project: project,
+ note: note,
+ commit_id: raw_data.commit_id,
+ line_code: line_code,
+ author_id: author_id,
+ created_at: raw_data.created_at,
+ updated_at: raw_data.updated_at
+ }
+ end
+
+ private
+
+ def author
+ raw_data.user.login
+ end
+
+ def author_id
+ gl_user_id(raw_data.user.id) || project.creator_id
+ end
+
+ def body
+ raw_data.body || ""
+ end
+
+ def line_code
+ if on_diff?
+ Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0)
+ end
+ end
+
+ def on_diff?
+ raw_data.path && raw_data.position
+ end
+
+ def note
+ formatter.author_line(author) + body
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index b5720f6e2cb..18929b9113b 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -1,6 +1,8 @@
module Gitlab
module GithubImport
class Importer
+ include Gitlab::ShellAdapter
+
attr_reader :project, :client
def initialize(project)
@@ -12,39 +14,76 @@ module Gitlab
end
def execute
- #Issues && Comments
+ import_issues && import_pull_requests && import_wiki
+ end
+
+ private
+
+ def import_issues
client.list_issues(project.import_source, state: :all,
sort: :created,
- direction: :asc).each do |issue|
- if issue.pull_request.nil?
-
- body = @formatter.author_line(issue.user.login)
- body += issue.body || ""
+ direction: :asc).each do |raw_data|
+ gh_issue = IssueFormatter.new(project, raw_data)
- if issue.comments > 0
- body += @formatter.comments_header
+ if gh_issue.valid?
+ issue = Issue.create!(gh_issue.attributes)
- client.issue_comments(project.import_source, issue.number).each do |c|
- body += @formatter.comment(c.user.login, c.created_at, c.body)
- end
+ if gh_issue.has_comments?
+ import_comments(gh_issue.number, issue)
end
+ end
+ end
+
+ true
+ rescue ActiveRecord::RecordInvalid
+ false
+ end
+
+ def import_pull_requests
+ client.pull_requests(project.import_source, state: :all,
+ sort: :created,
+ direction: :asc).each do |raw_data|
+ pull_request = PullRequestFormatter.new(project, raw_data)
- project.issues.create!(
- description: body,
- title: issue.title,
- state: issue.state == 'closed' ? 'closed' : 'opened',
- author_id: gl_user_id(project, issue.user.id)
- )
+ if !pull_request.cross_project? && pull_request.valid?
+ merge_request = MergeRequest.create!(pull_request.attributes)
+ import_comments(pull_request.number, merge_request)
+ import_comments_on_diff(pull_request.number, merge_request)
end
end
+
+ true
+ rescue ActiveRecord::RecordInvalid
+ false
end
- private
+ def import_comments(issue_number, noteable)
+ comments = client.issue_comments(project.import_source, issue_number)
+ create_comments(comments, noteable)
+ end
+
+ def import_comments_on_diff(pull_request_number, merge_request)
+ comments = client.pull_request_comments(project.import_source, pull_request_number)
+ create_comments(comments, merge_request)
+ end
+
+ def create_comments(comments, noteable)
+ comments.each do |raw_data|
+ comment = CommentFormatter.new(project, raw_data)
+ noteable.notes.create!(comment.attributes)
+ end
+ end
+
+ def import_wiki
+ unless project.wiki_enabled?
+ wiki = WikiFormatter.new(project)
+ gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+ project.update_attribute(:wiki_enabled, true)
+ end
- def gl_user_id(project, github_id)
- user = User.joins(:identities).
- find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s)
- (user && user.id) || project.creator_id
+ true
+ rescue Gitlab::Shell::Error
+ false
end
end
end
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
new file mode 100644
index 00000000000..1e3ba44f27c
--- /dev/null
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -0,0 +1,66 @@
+module Gitlab
+ module GithubImport
+ class IssueFormatter < BaseFormatter
+ def attributes
+ {
+ project: project,
+ title: raw_data.title,
+ description: description,
+ state: state,
+ author_id: author_id,
+ assignee_id: assignee_id,
+ created_at: raw_data.created_at,
+ updated_at: updated_at
+ }
+ end
+
+ def has_comments?
+ raw_data.comments > 0
+ end
+
+ def number
+ raw_data.number
+ end
+
+ def valid?
+ raw_data.pull_request.nil?
+ end
+
+ private
+
+ def assigned?
+ raw_data.assignee.present?
+ end
+
+ def assignee_id
+ if assigned?
+ gl_user_id(raw_data.assignee.id)
+ end
+ end
+
+ def author
+ raw_data.user.login
+ end
+
+ def author_id
+ gl_user_id(raw_data.user.id) || project.creator_id
+ end
+
+ def body
+ raw_data.body || ""
+ end
+
+ def description
+ @formatter.author_line(author) + body
+ end
+
+ def state
+ raw_data.state == 'closed' ? 'closed' : 'opened'
+ end
+
+ def updated_at
+ state == 'closed' ? raw_data.closed_at : raw_data.updated_at
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index 8c27ebd1ce8..474927069a5 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -20,7 +20,8 @@ module Gitlab
visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
import_type: "github",
import_source: repo.full_name,
- import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
+ import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"),
+ wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later
).execute
project.create_import_data(data: { "github_session" => session_data } )
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
new file mode 100644
index 00000000000..b7c47958cc7
--- /dev/null
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -0,0 +1,101 @@
+module Gitlab
+ module GithubImport
+ class PullRequestFormatter < BaseFormatter
+ def attributes
+ {
+ title: raw_data.title,
+ description: description,
+ source_project: source_project,
+ source_branch: source_branch.name,
+ target_project: target_project,
+ target_branch: target_branch.name,
+ state: state,
+ author_id: author_id,
+ assignee_id: assignee_id,
+ created_at: raw_data.created_at,
+ updated_at: updated_at
+ }
+ end
+
+ def cross_project?
+ source_repo.fork == true
+ end
+
+ def number
+ raw_data.number
+ end
+
+ def valid?
+ source_branch.present? && target_branch.present?
+ end
+
+ private
+
+ def assigned?
+ raw_data.assignee.present?
+ end
+
+ def assignee_id
+ if assigned?
+ gl_user_id(raw_data.assignee.id)
+ end
+ end
+
+ def author
+ raw_data.user.login
+ end
+
+ def author_id
+ gl_user_id(raw_data.user.id) || project.creator_id
+ end
+
+ def body
+ raw_data.body || ""
+ end
+
+ def description
+ formatter.author_line(author) + body
+ end
+
+ def source_project
+ project
+ end
+
+ def source_repo
+ raw_data.head.repo
+ end
+
+ def source_branch
+ source_project.repository.find_branch(raw_data.head.ref)
+ end
+
+ def target_project
+ project
+ end
+
+ def target_branch
+ target_project.repository.find_branch(raw_data.base.ref)
+ end
+
+ def state
+ @state ||= case true
+ when raw_data.state == 'closed' && raw_data.merged_at.present?
+ 'merged'
+ when raw_data.state == 'closed'
+ 'closed'
+ else
+ 'opened'
+ end
+ end
+
+ def updated_at
+ case state
+ when 'merged' then raw_data.merged_at
+ when 'closed' then raw_data.closed_at
+ else
+ raw_data.updated_at
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb
new file mode 100644
index 00000000000..6c592ff469c
--- /dev/null
+++ b/lib/gitlab/github_import/wiki_formatter.rb
@@ -0,0 +1,19 @@
+module Gitlab
+ module GithubImport
+ class WikiFormatter
+ attr_reader :project
+
+ def initialize(project)
+ @project = project
+ end
+
+ def path_with_namespace
+ "#{project.path_with_namespace}.wiki"
+ end
+
+ def import_url
+ project.import_url.sub(/\.git\z/, ".wiki.git")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index e24b94d6159..59926084d07 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -12,7 +12,7 @@ module Gitlab
end
def execute
- project_identifier = URI.encode(project.import_source, '/')
+ project_identifier = CGI.escape(project.import_source, '/')
#Issues && Comments
issues = client.issues(project_identifier)
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index c438a3d167b..da4435c7308 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -5,7 +5,7 @@
module Gitlab
module LDAP
class Access
- attr_reader :adapter, :provider, :user
+ attr_reader :provider, :user
def self.open(user, &block)
Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter|
@@ -32,20 +32,20 @@ module Gitlab
end
def allowed?
- if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
+ if ldap_user
return true unless ldap_config.active_directory
# Block user in GitLab if he/she was blocked in AD
if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter)
- user.block
+ user.ldap_block
false
else
- user.activate if user.blocked? && !ldap_config.block_auto_created_users
+ user.activate if user.ldap_blocked?
true
end
else
# Block the user if they no longer exist in LDAP/AD
- user.block
+ user.ldap_block
false
end
rescue
@@ -59,6 +59,10 @@ module Gitlab
def ldap_config
Gitlab::LDAP::Config.new(provider)
end
+
+ def ldap_user
+ @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
+ end
end
end
end
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 577a890a7d9..df65179bfea 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -70,19 +70,25 @@ module Gitlab
end
def ldap_search(*args)
- results = ldap.search(*args)
+ # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead.
+ Timeout.timeout(config.timeout) do
+ results = ldap.search(*args)
- if results.nil?
- response = ldap.get_operation_result
+ if results.nil?
+ response = ldap.get_operation_result
- unless response.code.zero?
- Rails.logger.warn("LDAP search error: #{response.message}")
- end
+ unless response.code.zero?
+ Rails.logger.warn("LDAP search error: #{response.message}")
+ end
- []
- else
- results
+ []
+ else
+ results
+ end
end
+ rescue Timeout::Error
+ Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds")
+ []
end
end
end
diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb
index 101a3285f4b..aff7ccb157f 100644
--- a/lib/gitlab/ldap/config.rb
+++ b/lib/gitlab/ldap/config.rb
@@ -88,6 +88,10 @@ module Gitlab
options['attributes']
end
+ def timeout
+ options['timeout'].to_i
+ end
+
protected
def base_config
Gitlab.config.ldap
diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb
index 8f3f43c0e91..699d8b9fc07 100644
--- a/lib/gitlab/markdown/pipeline.rb
+++ b/lib/gitlab/markdown/pipeline.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Gitlab
module Markdown
class Pipeline
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 9470633b065..88a265c6af2 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,36 +1,25 @@
module Gitlab
module Metrics
+ extend Gitlab::CurrentSettings
+
RAILS_ROOT = Rails.root.to_s
METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
PATH_REGEX = /^#{RAILS_ROOT}\/?/
- # Returns the current settings, ensuring we _always_ have a default set of
- # metrics settings (even during tests, when the migrations are lacking,
- # etc). This ensures the application is able to boot up even when the
- # migrations have not been executed.
def self.settings
- if ApplicationSetting.table_exists? and curr = ApplicationSetting.current
- curr
- else
- {
- metrics_pool_size: 16,
- metrics_timeout: 10,
- metrics_enabled: false,
- metrics_method_call_threshold: 10
- }
- end
- end
-
- def self.pool_size
- settings[:metrics_pool_size]
- end
-
- def self.timeout
- settings[:metrics_timeout]
+ @settings ||= {
+ enabled: current_application_settings[:metrics_enabled],
+ pool_size: current_application_settings[:metrics_pool_size],
+ timeout: current_application_settings[:metrics_timeout],
+ method_call_threshold: current_application_settings[:metrics_method_call_threshold],
+ host: current_application_settings[:metrics_host],
+ port: current_application_settings[:metrics_port],
+ sample_interval: current_application_settings[:metrics_sample_interval] || 15
+ }
end
def self.enabled?
- settings[:metrics_enabled]
+ settings[:enabled] || false
end
def self.mri?
@@ -41,43 +30,55 @@ module Gitlab
# This is memoized since this method is called for every instrumented
# method. Loading data from an external cache on every method call slows
# things down too much.
- @method_call_threshold ||= settings[:metrics_method_call_threshold]
+ @method_call_threshold ||= settings[:method_call_threshold]
end
def self.pool
@pool
end
- def self.hostname
- @hostname
- end
+ def self.submit_metrics(metrics)
+ prepared = prepare_metrics(metrics)
- # Returns a relative path and line number based on the last application call
- # frame.
- def self.last_relative_application_frame
- frame = caller_locations.find do |l|
- l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT)
+ pool.with do |connection|
+ prepared.each do |metric|
+ begin
+ connection.write_points([metric])
+ rescue StandardError
+ end
+ end
end
+ end
+
+ def self.prepare_metrics(metrics)
+ metrics.map do |hash|
+ new_hash = hash.symbolize_keys
- if frame
- return frame.path.sub(PATH_REGEX, ''), frame.lineno
- else
- return nil, nil
+ new_hash[:tags].each do |key, value|
+ if value.blank?
+ new_hash[:tags].delete(key)
+ else
+ new_hash[:tags][key] = escape_value(value)
+ end
+ end
+
+ new_hash
end
end
- @hostname = Socket.gethostname
+ def self.escape_value(value)
+ value.to_s.gsub('=', '\\=')
+ end
# When enabled this should be set before being used as the usual pattern
# "@foo ||= bar" is _not_ thread-safe.
if enabled?
- @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
- host = settings[:metrics_host]
- db = settings[:metrics_database]
- user = settings[:metrics_username]
- pw = settings[:metrics_password]
+ @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
+ host = settings[:host]
+ port = settings[:port]
- InfluxDB::Client.new(db, host: host, username: user, password: pw)
+ InfluxDB::Client.
+ new(udp: { host: host, port: port })
end
end
end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 06fc2f25948..d9fce2e6758 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -123,6 +123,8 @@ module Gitlab
duration = (Time.now - start) * 1000.0
if duration >= Gitlab::Metrics.method_call_threshold
+ trans.increment(:method_duration, duration)
+
trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
{ duration: duration },
method: #{label.inspect})
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index f592f4e571f..7ea9555cc8c 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -17,16 +17,10 @@ module Gitlab
# Returns a Hash in a format that can be directly written to InfluxDB.
def to_hash
{
- series: @series,
- tags: @tags.merge(
- hostname: Metrics.hostname,
- ruby_engine: RUBY_ENGINE,
- ruby_version: RUBY_VERSION,
- gitlab_version: Gitlab::VERSION,
- process_type: Sidekiq.server? ? 'sidekiq' : 'rails'
- ),
+ series: @series,
+ tags: @tags,
values: @values,
- timestamp: @created_at.to_i
+ timestamp: @created_at.to_i * 1_000_000_000
}
end
end
diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
deleted file mode 100644
index 481aca56efb..00000000000
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-module Gitlab
- module Metrics
- # Class for producing SQL queries with sensitive data stripped out.
- class ObfuscatedSQL
- REPLACEMENT = /
- \d+(\.\d+)? # integers, floats
- | '.+?' # single quoted strings
- | \/.+?(?<!\\)\/ # regexps (including escaped slashes)
- /x
-
- MYSQL_REPLACEMENTS = /
- ".+?" # double quoted strings
- /x
-
- # Regex to replace consecutive placeholders with a single one indicating
- # the length. This can be useful when a "IN" statement uses thousands of
- # IDs (storing this would just be a waste of space).
- CONSECUTIVE = /(\?(\s*,\s*)?){2,}/
-
- # sql - The raw SQL query as a String.
- def initialize(sql)
- @sql = sql
- end
-
- # Returns a new, obfuscated SQL query.
- def to_s
- regex = REPLACEMENT
-
- if Gitlab::Database.mysql?
- regex = Regexp.union(regex, MYSQL_REPLACEMENTS)
- end
-
- sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match|
- "#{match.count(',') + 1} values"
- end
-
- # InfluxDB escapes double quotes upon output, so lets get rid of them
- # whenever we can.
- if Gitlab::Database.postgresql?
- sql = sql.delete('"')
- end
-
- sql
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 5c0587c4c51..6f179789d3e 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -32,17 +32,15 @@ module Gitlab
def transaction_from_env(env)
trans = Transaction.new
- trans.add_tag(:request_method, env['REQUEST_METHOD'])
- trans.add_tag(:request_uri, env['REQUEST_URI'])
+ trans.set(:request_uri, env['REQUEST_URI'])
+ trans.set(:request_method, env['REQUEST_METHOD'])
trans
end
def tag_controller(trans, env)
- controller = env[CONTROLLER_KEY]
- label = "#{controller.class.name}##{controller.action_name}"
-
- trans.add_tag(:action, label)
+ controller = env[CONTROLLER_KEY]
+ trans.action = "#{controller.class.name}##{controller.action_name}"
end
end
end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 828ee1f8c62..fc709222a9b 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -7,9 +7,14 @@ module Gitlab
# statistics, etc.
class Sampler
# interval - The sampling interval in seconds.
- def initialize(interval = 15)
- @interval = interval
- @metrics = []
+ def initialize(interval = Metrics.settings[:sample_interval])
+ interval_half = interval.to_f / 2
+
+ @interval = interval
+ @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+ @last_step = nil
+
+ @metrics = []
@last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
@last_major_gc = Delta.new(GC.stat[:major_gc_count])
@@ -26,7 +31,7 @@ module Gitlab
Thread.current.abort_on_exception = true
loop do
- sleep(@interval)
+ sleep(sleep_interval)
sample
end
@@ -46,16 +51,15 @@ module Gitlab
end
def flush
- MetricsWorker.perform_async(@metrics.map(&:to_hash))
+ Metrics.submit_metrics(@metrics.map(&:to_hash))
end
def sample_memory_usage
- @metrics << Metric.new('memory_usage', value: System.memory_usage)
+ add_metric('memory_usage', value: System.memory_usage)
end
def sample_file_descriptors
- @metrics << Metric.
- new('file_descriptors', value: System.file_descriptor_count)
+ add_metric('file_descriptors', value: System.file_descriptor_count)
end
if Metrics.mri?
@@ -69,7 +73,7 @@ module Gitlab
counts['Symbol'] = Symbol.all_symbols.length
counts.each do |name, count|
- @metrics << Metric.new('object_counts', { count: count }, type: name)
+ add_metric('object_counts', { count: count }, type: name)
end
end
else
@@ -91,7 +95,34 @@ module Gitlab
stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count]
- @metrics << Metric.new('gc_statistics', stats)
+ add_metric('gc_statistics', stats)
+ end
+
+ def add_metric(series, values, tags = {})
+ prefix = sidekiq? ? 'sidekiq_' : 'rails_'
+
+ @metrics << Metric.new("#{prefix}#{series}", values, tags)
+ end
+
+ def sidekiq?
+ Sidekiq.server?
+ end
+
+ # Returns the sleep interval with a random adjustment.
+ #
+ # The random adjustment is put in place to ensure we:
+ #
+ # 1. Don't generate samples at the exact same interval every time (thus
+ # potentially missing anything that happens in between samples).
+ # 2. Don't sample data at the same interval two times in a row.
+ def sleep_interval
+ while step = @interval_steps.sample
+ if step != @last_step
+ @last_step = step
+
+ return @interval + @last_step
+ end
+ end
end
end
end
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index ec10707d1fb..fd98aa3412e 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -5,26 +5,14 @@ module Gitlab
# This middleware is intended to be used as a server-side middleware.
class SidekiqMiddleware
def call(worker, message, queue)
- # We don't want to track the MetricsWorker itself as otherwise we'll end
- # up in an infinite loop.
- if worker.class == MetricsWorker
- yield
- return
- end
-
- trans = Transaction.new
+ trans = Transaction.new("#{worker.class.name}#perform")
begin
trans.run { yield }
ensure
- tag_worker(trans, worker)
trans.finish
end
end
-
- def tag_worker(trans, worker)
- trans.add_tag(:action, "#{worker.class.name}#perform")
- end
end
end
end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 7e0dcf99d92..2e9dd4645e3 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -19,6 +19,7 @@ module Gitlab
values = values_for(event)
tags = tags_for(event)
+ current_transaction.increment(:view_duration, event.duration)
current_transaction.add_metric(SERIES, values, tags)
end
@@ -32,16 +33,8 @@ module Gitlab
def tags_for(event)
path = relative_path(event.payload[:identifier])
- tags = { view: path }
- file, line = Metrics.last_relative_application_frame
-
- if file and line
- tags[:file] = file
- tags[:line] = line
- end
-
- tags
+ { view: path }
end
def current_transaction
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index d947c128ce2..8008b3bc895 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -1,44 +1,18 @@
module Gitlab
module Metrics
module Subscribers
- # Class for tracking raw SQL queries.
- #
- # Queries are obfuscated before being logged to ensure no private data is
- # exposed via InfluxDB/Grafana.
+ # Class for tracking the total query duration of a transaction.
class ActiveRecord < ActiveSupport::Subscriber
attach_to :active_record
- SERIES = 'sql_queries'
-
def sql(event)
return unless current_transaction
- values = values_for(event)
- tags = tags_for(event)
-
- current_transaction.add_metric(SERIES, values, tags)
+ current_transaction.increment(:sql_duration, event.duration)
end
private
- def values_for(event)
- { duration: event.duration }
- end
-
- def tags_for(event)
- sql = ObfuscatedSQL.new(event.payload[:sql]).to_s
- tags = { sql: sql }
-
- file, line = Metrics.last_relative_application_frame
-
- if file and line
- tags[:file] = file
- tags[:line] = line
- end
-
- tags
- end
-
def current_transaction
Transaction.current
end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 568f9d6ae0c..2578ddc49f4 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,45 +4,64 @@ module Gitlab
class Transaction
THREAD_KEY = :_gitlab_metrics_transaction
- SERIES = 'transactions'
+ attr_reader :tags, :values
- attr_reader :uuid, :tags
+ attr_accessor :action
def self.current
Thread.current[THREAD_KEY]
end
- # name - The name of this transaction as a String.
- def initialize
+ # action - A String describing the action performed, usually the class
+ # plus method name.
+ def initialize(action = nil)
@metrics = []
- @uuid = SecureRandom.uuid
@started_at = nil
@finished_at = nil
- @tags = {}
+ @values = Hash.new(0)
+ @tags = {}
+ @action = action
+
+ @memory_before = 0
+ @memory_after = 0
end
def duration
@finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0
end
+ def allocated_memory
+ @memory_after - @memory_before
+ end
+
def run
Thread.current[THREAD_KEY] = self
- @started_at = Time.now
+ @memory_before = System.memory_usage
+ @started_at = Time.now
yield
ensure
- @finished_at = Time.now
+ @memory_after = System.memory_usage
+ @finished_at = Time.now
Thread.current[THREAD_KEY] = nil
end
def add_metric(series, values, tags = {})
- tags = tags.merge(transaction_id: @uuid)
+ prefix = sidekiq? ? 'sidekiq_' : 'rails_'
+
+ @metrics << Metric.new("#{prefix}#{series}", values, tags)
+ end
+
+ def increment(name, value)
+ @values[name] += value
+ end
- @metrics << Metric.new(series, values, tags)
+ def set(name, value)
+ @values[name] = value
end
def add_tag(key, value)
@@ -55,11 +74,29 @@ module Gitlab
end
def track_self
- add_metric(SERIES, { duration: duration }, @tags)
+ values = { duration: duration, allocated_memory: allocated_memory }
+
+ @values.each do |name, value|
+ values[name] = value
+ end
+
+ add_metric('transactions', values, @tags)
end
def submit
- MetricsWorker.perform_async(@metrics.map(&:to_hash))
+ metrics = @metrics.map do |metric|
+ hash = metric.to_hash
+
+ hash[:tags][:action] ||= @action if @action
+
+ hash
+ end
+
+ Metrics.submit_metrics(metrics)
+ end
+
+ def sidekiq?
+ Sidekiq.server?
end
end
end
diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb
new file mode 100644
index 00000000000..70e7f25d518
--- /dev/null
+++ b/lib/gitlab/recaptcha.rb
@@ -0,0 +1,14 @@
+module Gitlab
+ module Recaptcha
+ def self.load_configurations!
+ if current_application_settings.recaptcha_enabled
+ ::Recaptcha.configure do |config|
+ config.public_key = current_application_settings.recaptcha_site_key
+ config.private_key = current_application_settings.recaptcha_private_key
+ end
+
+ true
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index be795649e59..4d830aa45e1 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -1,5 +1,3 @@
-require 'banzai'
-
module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor < Banzai::ReferenceExtractor
@@ -19,7 +17,7 @@ module Gitlab
super(text, context.merge(project: project))
end
- %i(user label merge_request snippet commit commit_range).each do |type|
+ %i(user label milestone merge_request snippet commit commit_range).each do |type|
define_method("#{type}s") do
@references[type] ||= references(type, reference_context)
end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 0469c5a61c3..2dc2953e328 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -431,7 +431,7 @@ namespace :gitlab do
try_fixing_it(
"sudo chmod -R ug+rwX,o-rwx #{repo_base_path}",
"sudo chmod -R ug-s #{repo_base_path}",
- "find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
+ "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s"
)
for_more_information(
see_installation_guide_section "GitLab Shell"
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index ebe516ec879..8c63877e51c 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -2,6 +2,8 @@ module Gitlab
class TaskAbortedByUserError < StandardError; end
end
+String.disable_colorization = true unless STDOUT.isatty
+
namespace :gitlab do
# Ask if the user wants to continue
diff --git a/lib/version_check.rb b/lib/version_check.rb
index ea23344948c..91ad07feee5 100644
--- a/lib/version_check.rb
+++ b/lib/version_check.rb
@@ -13,6 +13,6 @@ class VersionCheck
end
def host
- 'https://version.gitlab.com/check.png'
+ 'https://version.gitlab.com/check.svg'
end
end