diff options
Diffstat (limited to 'lib')
33 files changed, 618 insertions, 117 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 592100a7045..231840148d9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -64,7 +64,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch does not exist") unless @branch + not_found!("Branch") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4c49442bf8b..60b9f5e0ece 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -170,6 +170,10 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic + + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user]) + end end class MergeRequest < ProjectEntity @@ -183,6 +187,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user]) + end end class MergeRequestChanges < MergeRequest @@ -204,7 +212,7 @@ module API expose :note, as: :body expose :attachment_identifier, as: :attachment expose :author, using: Entities::UserBasic - expose :created_at + expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type # upvote? and downvote? are deprecated, always return false @@ -255,14 +263,19 @@ module API expose :id, :path, :kind end - class ProjectAccess < Grape::Entity + class Member < Grape::Entity expose :access_level - expose :notification_level + expose :notification_level do |member, options| + if member.notification_setting + NotificationSetting.levels[member.notification_setting.level] + end + end end - class GroupAccess < Grape::Entity - expose :access_level - expose :notification_level + class ProjectAccess < Member + end + + class GroupAccess < Member end class ProjectService < Grape::Entity @@ -293,6 +306,7 @@ module API class Label < Grape::Entity expose :name, :color, :description + expose :open_issues_count, :closed_issues_count, :open_merge_requests_count end class Compare < Grape::Entity diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c165de21a75..91e420832f3 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,8 +23,10 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group # Example Request: # POST /groups post do @@ -42,6 +44,28 @@ module API end end + # Update group. Available only for users who can administrate groups. + # + # Parameters: + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # Example Request: + # PUT /groups/:id + put ':id' do + group = find_group(params[:id]) + authorize! :admin_group, group + + attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + + if ::Groups::UpdateService.new(group, current_user, attrs).execute + present group, with: Entities::GroupDetail + else + render_validation_error!(group) + end + end + # Get a single group, with containing projects # # Parameters: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4921ae99e78..5bbf721321d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,7 @@ module API if can?(current_user, :read_group, group) group else - forbidden!("#{current_user.username} lacks sufficient "\ - "access to #{group.name}") + not_found!('Group') end end @@ -241,6 +240,10 @@ module API render_api_error!('413 Request Entity Too Large', 413) end + def not_modified! + render_api_error!('304 Not Modified', 304) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1fee1dee1a6..4cdecadfe0f 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -55,7 +55,7 @@ module API issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -92,7 +92,7 @@ module API end issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end # Get a single project issue @@ -105,7 +105,7 @@ module API get ":id/issues/:issue_id" do @issue = user_project.issues.find(params[:issue_id]) not_found! unless can?(current_user, :read_issue, @issue) - present @issue, with: Entities::Issue + present @issue, with: Entities::Issue, current_user: current_user end # Create a new project issue @@ -149,7 +149,7 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end @@ -189,12 +189,35 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end end + # Move an existing issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # to_project_id (required) - The ID of the new project + # Example Request: + # POST /projects/:id/issues/:issue_id/move + post ':id/issues/:issue_id/move' do + required_attributes! [:to_project_id] + + issue = user_project.issues.find(params[:issue_id]) + new_project = Project.find(params[:to_project_id]) + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: Entities::Issue, current_user: current_user + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + # # Delete a project issue # # Parameters: @@ -208,6 +231,42 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + # Subscribes to a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # POST /projects/:id/issues/:issue_id/subscription + post ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + not_modified! + else + issue.toggle_subscription(current_user) + present issue, with: Entities::Issue, current_user: current_user + end + end + + # Unsubscribes from a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # DELETE /projects/:id/issues/:issue_id/subscription + delete ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + issue.unsubscribe(current_user) + present issue, with: Entities::Issue, current_user: current_user + else + not_modified! + end + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 93052fba06b..7e78609ecb9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -56,7 +56,7 @@ module API end merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user end # Create MR @@ -94,7 +94,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -130,7 +130,7 @@ module API authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Show MR commits @@ -162,7 +162,7 @@ module API merge_request = user_project.merge_requests. find(params[:merge_request_id]) authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end # Update MR @@ -204,7 +204,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -246,7 +246,7 @@ module API execute(merge_request) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Cancel Merge if Merge When build succeeds is enabled @@ -325,7 +325,43 @@ module API get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user + end + + # Subscribes to a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # POST /projects/:id/issues/:merge_request_id/subscription + post "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + not_modified! + else + merge_request.toggle_subscription(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + end + end + + # Unsubscribes from a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # DELETE /projects/:id/merge_requests/:merge_request_id/subscription + delete "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + merge_request.unsubscribe(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + else + not_modified! + end end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index c5cd73943fb..84b4d4cdd6d 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -3,17 +3,35 @@ module API class Milestones < Grape::API before { authenticate! } + helpers do + def filter_milestones_state(milestones, state) + case state + when 'active' then milestones.active + when 'closed' then milestones.closed + else milestones + end + end + end + resource :projects do # Get a list of project milestones # # Parameters: - # id (required) - The ID of a project + # id (required) - The ID of a project + # state (optional) - Return "active" or "closed" milestones # Example Request: # GET /projects/:id/milestones + # GET /projects/:id/milestones?iid=42 + # GET /projects/:id/milestones?state=active + # GET /projects/:id/milestones?state=closed get ":id/milestones" do authorize! :read_milestone, user_project - present paginate(user_project.milestones), with: Entities::Milestone + milestones = user_project.milestones + milestones = filter_milestones_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + + present paginate(milestones), with: Entities::Milestone end # Get a single project milestone @@ -87,7 +105,7 @@ module API authorize! :read_milestone, user_project @milestone = user_project.milestones.find(params[:milestone_id]) - present paginate(@milestone.issues), with: Entities::Issue + present paginate(@milestone.issues), with: Entities::Issue, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..a1c98f5e8ff 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -112,6 +112,23 @@ module API end end + # Delete a +noteable+ note + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue, MR, or snippet + # node_id (required) - The ID of a note + # Example Request: + # DELETE /projects/:id/issues/:noteable_id/notes/:note_id + # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id + delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DeleteService.new(user_project, current_user).execute(note) + + present note, with: Entities::Note + end end end end diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index c756bb479fc..4aefdf319c6 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -93,12 +93,17 @@ module API # Example Request: # DELETE /projects/:id/members/:user_id delete ":id/members/:user_id" do - authorize! :admin_project, user_project project_member = user_project.project_members.find_by(user_id: params[:user_id]) - unless project_member.nil? - project_member.destroy - else + + unless current_user.can?(:admin_project, user_project) || + current_user.can?(:destroy_project_member, project_member) + forbidden! + end + + if project_member.nil? { message: "Access revoked", id: params[:user_id].to_i } + else + project_member.destroy end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 24b31005475..cc2c7a0c503 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -272,6 +272,40 @@ module API present user_project, with: Entities::Project end + # Star project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/star + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + end + end + + # Unstar project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id/star + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + else + not_modified! + end + end + # Remove project # # Parameters: diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0d0f0d4616d..62161aadb9a 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -98,7 +98,6 @@ module API authorize! :download_code, user_project begin - RepositoryArchiveCacheWorker.perform_async header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) rescue not_found!('File') diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 731a68082ba..d1a10479e44 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -19,15 +19,15 @@ module API # Get a single repository tag # # Parameters: - # id (required) - The ID of a project + # id (required) - The ID of a project # tag_name (required) - The name of the tag # Example Request: # GET /projects/:id/repository/tags/:tag_name - get ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag - present tag, with: Entities::RepoTag, project: user_project + present tag, with: Entities::RepoTag, project: user_project end # Create tag diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index f21dbef216c..b8962379cb5 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -119,7 +119,7 @@ module Banzai elsif element_node?(node) yield_valid_link(node) do |link, text| - if ref_pattern && link =~ /\A#{ref_pattern}/ + if ref_pattern && link =~ /\A#{ref_pattern}\z/ replace_link_node_with_href(node, link) do object_link_filter(link, ref_pattern, link_text: text) end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 7ce26db1b90..d08267a9d6c 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -118,7 +118,7 @@ module Banzai end if path - content_tag(:img, nil, src: path) + content_tag(:img, nil, src: path, class: 'gfm') end end @@ -144,12 +144,18 @@ module Banzai # if it is not. def process_page_link_tag(parts) if parts.size == 1 - url = parts[0].strip + reference = parts[0].strip else - name, url = *parts.compact.map(&:strip) + name, reference = *parts.compact.map(&:strip) end - content_tag(:a, name || url, href: url) + if url?(reference) + href = reference + else + href = ::File.join(project_wiki_base_path, reference) + end + + content_tag(:a, name || reference, href: href, class: 'gfm') end def project_wiki diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb new file mode 100644 index 00000000000..06d10c98501 --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -0,0 +1,56 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" relative links to files in a repository. + # + # Context options: + # :project_wiki + class WikiLinkFilter < HTML::Pipeline::Filter + + def call + return doc unless project_wiki? + + doc.search('a:not(.gfm)').each do |el| + process_link_attr el.attribute('href') + end + + doc + end + + protected + + def project_wiki? + !context[:project_wiki].nil? + end + + def process_link_attr(html_attr) + return if html_attr.blank? || file_reference?(html_attr) + + uri = URI(html_attr.value) + if uri.relative? && uri.path.present? + html_attr.value = rebuild_wiki_uri(uri).to_s + end + rescue URI::Error + # noop + end + + def rebuild_wiki_uri(uri) + uri.path = ::File.join(project_wiki_base_path, uri.path) + uri + end + + def file_reference?(html_attr) + !File.extname(html_attr.value).blank? + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 0b5a9e0b2b8..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -2,8 +2,10 @@ module Banzai module Pipeline class WikiPipeline < FullPipeline def self.filters - @filters ||= super.insert_after(Filter::TableOfContentsFilter, - Filter::GollumTagsFilter) + @filters ||= begin + super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) + .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ae714c87dc5..c14a9c4c722 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -19,8 +19,10 @@ module Banzai cache_key = full_cache_key(cache_key, context[:pipeline]) if cache_key - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + Gitlab::Metrics.measure(:banzai_cached_render) do + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) + end end else cacheless_render(text, context) @@ -64,13 +66,15 @@ module Banzai private def self.cacheless_render(text, context = {}) - result = render_result(text, context) + Gitlab::Metrics.measure(:banzai_cacheless_render) do + result = render_result(text, context) - output = result[:output] - if output.respond_to?(:to_html) - output.to_html - else - output.to_s + output = result[:output] + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 6108697bc20..7479e729db1 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,7 @@ require 'gitlab/git' module Gitlab + def self.com? + Gitlab.config.gitlab.url == 'https://gitlab.com' + end end diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb index 28a2391dbf8..e5e9fab3f5c 100644 --- a/lib/gitlab/badge/build.rb +++ b/lib/gitlab/badge/build.rb @@ -4,14 +4,15 @@ module Gitlab # Build badge # class Build + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + def initialize(project, ref) + @project, @ref = project, ref @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) end - def to_s - @image[:name].sub(/\.svg$/, '') - end - def type 'image/svg+xml' end @@ -19,6 +20,27 @@ module Gitlab def data File.read(@image[:path]) end + + def to_s + @image[:name].sub(/\.svg$/, '') + end + + def to_html + link_to(image_tag(image_url, alt: 'build status'), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index c73eca832d7..ffe49364379 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -43,18 +43,15 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + Gitlab::Redis.with do |redis| + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end end # No #cancel method. See comments above! private - def redis - # Maybe someday we want to use a connection pool... - @redis ||= Redis.new(url: Gitlab::RedisConfig.url) - end - def redis_key "gitlab:exclusive_lease:#{@key}" end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index da4435c7308..f2b649e50a2 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -33,7 +33,10 @@ module Gitlab def allowed? if ldap_user - return true unless ldap_config.active_directory + unless ldap_config.active_directory + user.activate if user.ldap_blocked? + return true + end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 4a3f47b5a95..484970c5a10 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -74,28 +74,46 @@ module Gitlab # # Example: # - # Gitlab::Metrics.measure(:find_by_username_timings) do + # Gitlab::Metrics.measure(:find_by_username_duration) do # User.find_by_username(some_username) # end # - # series - The name of the series to store the data in. - # values - A Hash containing extra values to add to the metric. - # tags - A Hash containing extra tags to add to the metric. + # name - The name of the field to store the execution time in. # # Returns the value yielded by the supplied block. - def self.measure(series, values = {}, tags = {}) - return yield unless Transaction.current + def self.measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time - start = Time.now.to_f retval = yield - duration = (Time.now.to_f - start) * 1000.0 - values = values.merge(duration: duration) - Transaction.current.add_metric(series, values, tags) + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) retval end + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def self.tag_transaction(name, value) + trans = current_transaction + + trans.add_tag(name, value) if trans + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -107,5 +125,11 @@ module Gitlab new(udp: { host: host, port: port }) end end + + private + + def self.current_transaction + Transaction.current + end end end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 7ea9555cc8c..1cd1ca30f70 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric + JITTER_RANGE = 0.000001..0.001 + attr_reader :series, :values, :tags, :created_at # series - The name of the series (as a String) to store the metric in. @@ -16,11 +18,29 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash + # InfluxDB overwrites an existing point if a new point has the same + # series, tag set, and timestamp. In a highly concurrent environment + # this means that using the number of seconds since the Unix epoch is + # inevitably going to collide with another timestamp. For example, two + # Rails requests processed by different processes may end up generating + # metrics using the _exact_ same timestamp (in seconds). + # + # Due to the way InfluxDB is set up there's no solution to this problem, + # all we can do is lower the amount of collisions. We do this by using + # Time#to_f which returns the seconds as a Float providing greater + # accuracy. We then add a small random value that is large enough to + # distinguish most timestamps but small enough to not alter the amount + # of seconds. + # + # See https://gitlab.com/gitlab-com/operations/issues/175 for more + # information. + time = @created_at.to_f + rand(JITTER_RANGE) + { series: @series, tags: @tags, values: @values, - timestamp: @created_at.to_i * 1_000_000_000 + timestamp: (time * 1_000_000_000).to_i } end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb new file mode 100644 index 00000000000..49e5f86e6e6 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -0,0 +1,39 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in Rails cache calls + class RailsCache < ActiveSupport::Subscriber + attach_to :active_support + + def cache_read(event) + increment(:cache_read_duration, event.duration) + end + + def cache_write(event) + increment(:cache_write_duration, event.duration) + end + + def cache_delete(event) + increment(:cache_delete_duration, event.duration) + end + + def cache_exist?(event) + increment(:cache_exists_duration, event.duration) + end + + def increment(key, duration) + return unless current_transaction + + current_transaction.increment(:cache_duration, duration) + current_transaction.increment(key, duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 83371265278..a7d183b2f94 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -30,6 +30,17 @@ module Gitlab 0 end end + + # THREAD_CPUTIME is not supported on OS X + if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) + def self.cpu_time + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + end + else + def self.cpu_time + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + end + end end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 832fb08a526..356e96fcbab 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -54,6 +54,12 @@ module Gitlab @user ||= build_new_user end + if external_provider? && @user + @user.external = true + elsif @user + @user.external = false + end + @user end @@ -113,6 +119,10 @@ module Gitlab end end + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..5c352c96de5 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,50 @@ +module Gitlab + class Redis + CACHE_NAMESPACE = 'cache:gitlab' + SESSION_NAMESPACE = 'session:gitlab' + SIDEKIQ_NAMESPACE = 'resque:gitlab' + + attr_reader :url + + # To be thread-safe we must be careful when writing the class instance + # variables @url and @pool. Because @pool depends on @url we need two + # mutexes to prevent deadlock. + URL_MUTEX = Mutex.new + POOL_MUTEX = Mutex.new + private_constant :URL_MUTEX, :POOL_MUTEX + + def self.url + @url || URL_MUTEX.synchronize { @url = new.url } + end + + def self.with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(url: url) } + end + end + @pool.with { |redis| yield redis } + end + + def self.redis_store_options + url = new.url + redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb deleted file mode 100644 index 4949c6db539..00000000000 --- a/lib/gitlab/redis_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - class RedisConfig - attr_reader :url - - def self.url - new.url - end - - def self.redis_store_options - url = new.url - redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) - if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path - end - redis_config_hash - end - - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) - - @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] - end - end - end -end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb new file mode 100644 index 00000000000..32c1c9ec5bb --- /dev/null +++ b/lib/gitlab/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Saml + class AuthHash < Gitlab::OAuth::AuthHash + + def groups + get_raw(Gitlab::Saml::Config.groups) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + + end + end +end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb new file mode 100644 index 00000000000..0f40c00f547 --- /dev/null +++ b/lib/gitlab/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Saml + class Config + + class << self + def options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + + end + end +end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index b1e30110ef5..dba4bbfc899 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -18,7 +18,7 @@ module Gitlab @user ||= find_or_create_ldap_user end - if auto_link_saml_enabled? + if auto_link_saml_user? @user ||= find_by_email end @@ -26,6 +26,16 @@ module Gitlab @user ||= build_new_user end + if external_users_enabled? && @user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + @user.external = false + else + @user.external = true + end + end + @user end @@ -37,11 +47,24 @@ module Gitlab end end + def changed? + return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + protected - def auto_link_saml_enabled? + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end + + def external_users_enabled? + !Gitlab::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) + end end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 51e746ef923..2214f855200 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -4,18 +4,19 @@ namespace :cache do desc "GitLab | Clear redis cache" task :clear => :environment do - redis = Redis.new(url: Gitlab::RedisConfig.url) - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 7ec00a898fd..030ee8bafcb 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -5,12 +5,23 @@ namespace :gemojione do require 'json' dir = Gemojione.index.images_path + digests = [] + aliases = Hash.new { |hash, key| hash[key] = [] } + aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - digests = AwardEmoji.emojis.map do |name, emoji_hash| + JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| + aliases[real_name] << alias_name + end + + AwardEmoji.emojis.map do |name, emoji_hash| fpath = File.join(dir, "#{emoji_hash['unicode']}.png") digest = Digest::SHA256.file(fpath).hexdigest - { name: name, unicode: emoji_hash['unicode'], digest: digest } + digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + + aliases[name].each do |alias_name| + digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + end end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') |