diff options
Diffstat (limited to 'lib')
92 files changed, 1392 insertions, 539 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 592100a7045..231840148d9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -64,7 +64,7 @@ module API authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch does not exist") unless @branch + not_found!("Branch") unless @branch protected_branch = user_project.protected_branches.find_by(name: @branch.name) protected_branch.destroy if protected_branch diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 197e826e5bc..60b9f5e0ece 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -8,14 +8,14 @@ module API expose :id, :state, :avatar_url expose :web_url do |user, options| - Gitlab::Application.routes.url_helpers.user_url(user) + Gitlab::Routing.url_helpers.user_url(user) end end class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url end class Identity < Grape::Entity @@ -89,7 +89,7 @@ module API expose :avatar_url expose :web_url do |group, options| - Gitlab::Application.routes.url_helpers.group_url(group) + Gitlab::Routing.url_helpers.group_url(group) end end @@ -170,6 +170,10 @@ module API expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic + + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user]) + end end class MergeRequest < ProjectEntity @@ -183,6 +187,10 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user]) + end end class MergeRequestChanges < MergeRequest @@ -204,7 +212,7 @@ module API expose :note, as: :body expose :attachment_identifier, as: :attachment expose :author, using: Entities::UserBasic - expose :created_at + expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type # upvote? and downvote? are deprecated, always return false @@ -255,14 +263,19 @@ module API expose :id, :path, :kind end - class ProjectAccess < Grape::Entity + class Member < Grape::Entity expose :access_level - expose :notification_level + expose :notification_level do |member, options| + if member.notification_setting + NotificationSetting.levels[member.notification_setting.level] + end + end end - class GroupAccess < Grape::Entity - expose :access_level - expose :notification_level + class ProjectAccess < Member + end + + class GroupAccess < Member end class ProjectService < Grape::Entity @@ -292,7 +305,8 @@ module API end class Label < Grape::Entity - expose :name, :color + expose :name, :color, :description + expose :open_issues_count, :closed_issues_count, :open_merge_requests_count end class Compare < Grape::Entity @@ -334,7 +348,6 @@ module API expose :updated_at expose :home_page_url expose :default_branch_protection - expose :twitter_sharing_enabled expose :restricted_visibility_levels expose :max_attachment_size expose :session_expire_delay diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c165de21a75..91e420832f3 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,8 +23,10 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group # Example Request: # POST /groups post do @@ -42,6 +44,28 @@ module API end end + # Update group. Available only for users who can administrate groups. + # + # Parameters: + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # Example Request: + # PUT /groups/:id + put ':id' do + group = find_group(params[:id]) + authorize! :admin_group, group + + attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + + if ::Groups::UpdateService.new(group, current_user, attrs).execute + present group, with: Entities::GroupDetail + else + render_validation_error!(group) + end + end + # Get a single group, with containing projects # # Parameters: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4921ae99e78..5bbf721321d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,7 @@ module API if can?(current_user, :read_group, group) group else - forbidden!("#{current_user.username} lacks sufficient "\ - "access to #{group.name}") + not_found!('Group') end end @@ -241,6 +240,10 @@ module API render_api_error!('413 Request Entity Too Large', 413) end + def not_modified! + render_api_error!('304 Not Modified', 304) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e5ae88eb96f..8aa08fd5acc 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -55,7 +55,7 @@ module API issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -92,7 +92,7 @@ module API end issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user end # Get a single project issue @@ -105,23 +105,27 @@ module API get ":id/issues/:issue_id" do @issue = user_project.issues.find(params[:issue_id]) not_found! unless can?(current_user, :read_issue, @issue) - present @issue, with: Entities::Issue + present @issue, with: Entities::Issue, current_user: current_user end # Create a new project issue # # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue + # id (required) - The ID of a project + # title (required) - The title of an issue + # description (optional) - The description of an issue + # assignee_id (optional) - The ID of a user to assign issue # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue + # labels (optional) - The labels of an issue + # created_at (optional) - Date time string, ISO 8601 formatted # Example Request: # POST /projects/:id/issues post ":id/issues" do required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id] + + keys = [:title, :description, :assignee_id, :milestone_id] + keys << :created_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -145,7 +149,7 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end @@ -162,12 +166,15 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) + # updated_at (optional) - Date time string, ISO 8601 formatted # Example Request: # PUT /projects/:id/issues/:issue_id put ":id/issues/:issue_id" do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys << :updated_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -185,12 +192,35 @@ module API issue.add_labels_by_names(params[:labels].split(',')) end - present issue, with: Entities::Issue + present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) end end + # Move an existing issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # to_project_id (required) - The ID of the new project + # Example Request: + # POST /projects/:id/issues/:issue_id/move + post ':id/issues/:issue_id/move' do + required_attributes! [:to_project_id] + + issue = user_project.issues.find(params[:issue_id]) + new_project = Project.find(params[:to_project_id]) + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: Entities::Issue, current_user: current_user + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + # # Delete a project issue # # Parameters: @@ -204,6 +234,42 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + # Subscribes to a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # POST /projects/:id/issues/:issue_id/subscription + post ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + not_modified! + else + issue.toggle_subscription(current_user) + present issue, with: Entities::Issue, current_user: current_user + end + end + + # Unsubscribes from a project issue + # + # Parameters: + # id (required) - The ID of a project + # issue_id (required) - The ID of a project issue + # Example Request: + # DELETE /projects/:id/issues/:issue_id/subscription + delete ':id/issues/:issue_id/subscription' do + issue = user_project.issues.find(params[:issue_id]) + + if issue.subscribed?(current_user) + issue.unsubscribe(current_user) + present issue, with: Entities::Issue, current_user: current_user + else + not_modified! + end + end end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 78ca58ad0d1..4af6bef0fa7 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -17,17 +17,18 @@ module API # Creates a new label # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be created + # color (required) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # POST /projects/:id/labels post ':id/labels' do authorize! :admin_label, user_project required_attributes! [:name, :color] - attrs = attributes_for_keys [:name, :color] + attrs = attributes_for_keys [:name, :color, :description] label = user_project.find_label(attrs[:name]) conflict!('Label already exists') if label @@ -62,11 +63,12 @@ module API # Updates an existing label. At least one optional parameter is required. # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be deleted + # new_name (optional) - The new name of the label + # color (optional) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # PUT /projects/:id/labels put ':id/labels' do @@ -76,7 +78,7 @@ module API label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color] + attrs = attributes_for_keys [:new_name, :color, :description] if attrs.empty? render_api_error!('Required parameters "new_name" or "color" ' \ diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 93052fba06b..7e78609ecb9 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -56,7 +56,7 @@ module API end merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user end # Create MR @@ -94,7 +94,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -130,7 +130,7 @@ module API authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Show MR commits @@ -162,7 +162,7 @@ module API merge_request = user_project.merge_requests. find(params[:merge_request_id]) authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end # Update MR @@ -204,7 +204,7 @@ module API merge_request.add_labels_by_names(params[:labels].split(",")) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors end @@ -246,7 +246,7 @@ module API execute(merge_request) end - present merge_request, with: Entities::MergeRequest + present merge_request, with: Entities::MergeRequest, current_user: current_user end # Cancel Merge if Merge When build succeeds is enabled @@ -325,7 +325,43 @@ module API get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: Entities::Issue + present paginate(issues), with: Entities::Issue, current_user: current_user + end + + # Subscribes to a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # POST /projects/:id/issues/:merge_request_id/subscription + post "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + not_modified! + else + merge_request.toggle_subscription(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + end + end + + # Unsubscribes from a merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of a merge request + # Example Request: + # DELETE /projects/:id/merge_requests/:merge_request_id/subscription + delete "#{path}/subscription" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + if merge_request.subscribed?(current_user) + merge_request.unsubscribe(current_user) + present merge_request, with: Entities::MergeRequest, current_user: current_user + else + not_modified! + end end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index c5cd73943fb..84b4d4cdd6d 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -3,17 +3,35 @@ module API class Milestones < Grape::API before { authenticate! } + helpers do + def filter_milestones_state(milestones, state) + case state + when 'active' then milestones.active + when 'closed' then milestones.closed + else milestones + end + end + end + resource :projects do # Get a list of project milestones # # Parameters: - # id (required) - The ID of a project + # id (required) - The ID of a project + # state (optional) - Return "active" or "closed" milestones # Example Request: # GET /projects/:id/milestones + # GET /projects/:id/milestones?iid=42 + # GET /projects/:id/milestones?state=active + # GET /projects/:id/milestones?state=closed get ":id/milestones" do authorize! :read_milestone, user_project - present paginate(user_project.milestones), with: Entities::Milestone + milestones = user_project.milestones + milestones = filter_milestones_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + + present paginate(milestones), with: Entities::Milestone end # Get a single project milestone @@ -87,7 +105,7 @@ module API authorize! :read_milestone, user_project @milestone = user_project.milestones.find(params[:milestone_id]) - present paginate(@milestone.issues), with: Entities::Issue + present paginate(@milestone.issues), with: Entities::Issue, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 174473f5371..71a53e6f0d6 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -61,6 +61,7 @@ module API # id (required) - The ID of a project # noteable_id (required) - The ID of an issue or snippet # body (required) - The content of a note + # created_at (optional) - The date # Example Request: # POST /projects/:id/issues/:noteable_id/notes # POST /projects/:id/snippets/:noteable_id/notes @@ -73,6 +74,10 @@ module API noteable_id: params[noteable_id_str] } + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end + @note = ::Notes::CreateService.new(user_project, current_user, opts).execute if @note.valid? @@ -112,6 +117,23 @@ module API end end + # Delete a +noteable+ note + # + # Parameters: + # id (required) - The ID of a project + # noteable_id (required) - The ID of an issue, MR, or snippet + # node_id (required) - The ID of a note + # Example Request: + # DELETE /projects/:id/issues/:noteable_id/notes/:note_id + # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id + delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DeleteService.new(user_project, current_user).execute(note) + + present note, with: Entities::Note + end end end end diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index c756bb479fc..4aefdf319c6 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -93,12 +93,17 @@ module API # Example Request: # DELETE /projects/:id/members/:user_id delete ":id/members/:user_id" do - authorize! :admin_project, user_project project_member = user_project.project_members.find_by(user_id: params[:user_id]) - unless project_member.nil? - project_member.destroy - else + + unless current_user.can?(:admin_project, user_project) || + current_user.can?(:destroy_project_member, project_member) + forbidden! + end + + if project_member.nil? { message: "Access revoked", id: params[:user_id].to_i } + else + project_member.destroy end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6fcb5261e40..cc2c7a0c503 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -244,6 +244,68 @@ module API end end + # Archive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/archive + post ':id/archive' do + authorize!(:archive_project, user_project) + + user_project.archive! + + present user_project, with: Entities::Project + end + + # Unarchive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/unarchive + post ':id/unarchive' do + authorize!(:archive_project, user_project) + + user_project.unarchive! + + present user_project, with: Entities::Project + end + + # Star project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/star + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + end + end + + # Unstar project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id/star + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: Entities::Project + else + not_modified! + end + end + # Remove project # # Parameters: diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0d0f0d4616d..62161aadb9a 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -98,7 +98,6 @@ module API authorize! :download_code, user_project begin - RepositoryArchiveCacheWorker.perform_async header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) rescue not_found!('File') diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 2d8a9e51bb9..d1a10479e44 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -16,6 +16,20 @@ module API with: Entities::RepoTag, project: user_project end + # Get a single repository tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # GET /projects/:id/repository/tags/:tag_name + get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + tag = user_project.repository.find_tag(params[:tag_name]) + not_found!('Tag') unless tag + + present tag, with: Entities::RepoTag, project: user_project + end + # Create tag # # Parameters: diff --git a/lib/api/users.rb b/lib/api/users.rb index 13ab17c6904..0a14bac07c0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -58,6 +58,7 @@ module API # extern_uid - External authentication provider UID # provider - External provider # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # confirm - Require user confirmation - true (default) or false @@ -67,7 +68,7 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) user = User.build_user(attrs) @@ -106,6 +107,7 @@ module API # website_url - Website url # projects_limit - Limit projects each user can create # bio - Bio + # location - Location of the user # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # external - Flags the user as external - true or false(default) @@ -114,7 +116,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] user = User.find(params[:id]) not_found!('User') unless user diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 783fcfb61ad..5f8ff01b0a9 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -14,19 +14,29 @@ class AwardEmoji food_drink: "Food" }.with_indifferent_access + CATEGORY_ALIASES = { + symbols: "objects_symbols", + foods: "food_drink", + travel: "travel_places" + }.with_indifferent_access + def self.normilize_emoji_name(name) aliases[name] || name end def self.emoji_by_category unless @emoji_by_category - @emoji_by_category = {} + @emoji_by_category = Hash.new { |h, key| h[key] = [] } emojis.each do |emoji_name, data| data["name"] = emoji_name - @emoji_by_category[data["category"]] ||= [] - @emoji_by_category[data["category"]] << data + # Skip Fitzpatrick(tone) modifiers + next if data["category"] == "modifier" + + category = CATEGORY_ALIASES[data["category"]] || data["category"] + + @emoji_by_category[category] << data end @emoji_by_category = @emoji_by_category.sort.to_h @@ -48,4 +58,23 @@ class AwardEmoji JSON.parse(File.read(json_path)) end end + + # Returns an Array of Emoji names and their asset URLs. + def self.urls + @urls ||= begin + path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + prefix = Gitlab::Application.config.assets.prefix + digest = Gitlab::Application.config.assets.digest + + JSON.parse(File.read(path)).map do |hash| + if digest + fname = "#{hash['unicode']}-#{hash['digest']}" + else + fname = hash['unicode'] + end + + { name: hash['name'], path: "#{prefix}/#{fname}.png" } + end + end + end end diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index 905c4c0144e..3eb544dfef9 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,3 @@ -require 'active_support/core_ext/string/output_safety' - module Banzai module Filter def self.[](name) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 34c38913474..b8962379cb5 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -11,15 +11,19 @@ module Banzai end def self.object_name - object_class.name.underscore + @object_name ||= object_class.name.underscore end def self.object_sym - object_name.to_sym + @object_sym ||= object_name.to_sym end def self.data_reference - "data-#{object_name.dasherize}" + @data_reference ||= "data-#{object_name.dasherize}" + end + + def self.object_class_title + @object_title ||= object_class.name.titleize end # Public: Find references in text (like `!123` for merge requests) @@ -53,6 +57,10 @@ module Banzai self.class.object_sym end + def object_class_title + self.class.object_class_title + end + def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -62,36 +70,81 @@ module Banzai # Example: project.merge_requests.find end + def find_object_cached(project, id) + if RequestStore.active? + cache = find_objects_cache[object_class][project.id] + + get_or_set_cache(cache, id) { find_object(project, id) } + else + find_object(project, id) + end + end + + def project_from_ref_cache(ref) + if RequestStore.active? + cache = project_refs_cache + + get_or_set_cache(cache, ref) { project_from_ref(ref) } + else + project_from_ref(ref) + end + end + def url_for_object(object, project) # Implement in child class # Example: project_merge_request_url end - def call - if object_class.reference_pattern - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + def url_for_object_cached(object, project) + if RequestStore.active? + cache = url_for_object_cache[object_class][project.id] - # `[Issue](#123)`, which is turned into - # `<a href="#123">Issue</a>` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) - end + get_or_set_cache(cache, object) { url_for_object(object, project) } + else + url_for_object(object, project) end + end - if object_class.link_reference_pattern - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + def call + return doc if project.nil? + + ref_pattern = object_class.reference_pattern + link_pattern = object_class.link_reference_pattern - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + each_node do |node| + if text_node?(node) && ref_pattern + replace_text_when_pattern_matches(node, ref_pattern) do |content| + object_link_filter(content, ref_pattern) + end + + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if ref_pattern && link =~ /\A#{ref_pattern}\z/ + replace_link_node_with_href(node, link) do + object_link_filter(link, ref_pattern, link_text: text) + end + + next + end + + next unless link_pattern + + if link == text && text =~ /\A#{link_pattern}/ + replace_link_node_with_text(node, link) do + object_link_filter(text, link_pattern) + end + + next + end + + if link =~ /\A#{link_pattern}\z/ + replace_link_node_with_href(node, link) do + object_link_filter(link, link_pattern, link_text: text) + end + + next + end + end end end @@ -109,9 +162,9 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_text: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref(project_ref) + project = project_from_ref_cache(project_ref) - if project && object = find_object(project, id) + if project && object = find_object_cached(project, id) title = object_link_title(object) klass = reference_class(object_sym) @@ -121,8 +174,11 @@ module Banzai object_sym => object.id ) - url = matches[:url] if matches.names.include?("url") - url ||= url_for_object(object, project) + if matches.names.include?("url") && matches[:url] + url = matches[:url] + else + url = url_for_object_cached(object, project) + end text = link_text || object_link_text(object, matches) @@ -146,7 +202,7 @@ module Banzai end def object_link_title(object) - "#{object_class.name.titleize}: #{object.title}" + "#{object_class_title}: #{object.title}" end def object_link_text(object, matches) @@ -157,6 +213,32 @@ module Banzai text end + + private + + def project_refs_cache + RequestStore[:banzai_project_refs] ||= {} + end + + def find_objects_cache + RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end + + def url_for_object_cache + RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end + + def get_or_set_cache(cache, key) + if cache.key?(key) + cache[key] + else + cache[key] = yield + end + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 856f56fb175..fac7dad3243 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 470727ee312..b469ea0f626 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -43,7 +43,7 @@ module Banzai end def url_for_object(range, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 713a56ba949..bd88207326c 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -37,7 +37,7 @@ module Banzai end def url_for_object(commit, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 207437ba7cf..d25de900674 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,7 +1,3 @@ -require 'action_controller' -require 'gitlab_emoji' -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that replaces :emoji: with images. diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index edc26386903..37344b90576 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -35,15 +35,29 @@ module Banzai def call # Early return if the project isn't using an external tracker - return doc if project.nil? || project.default_issues_tracker? + return doc if project.nil? || default_issues_tracker? - replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| - issue_link_filter(content) - end + ref_pattern = ExternalIssue.reference_pattern + ref_start_pattern = /\A#{ref_pattern}\z/ + + each_node do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + issue_link_filter(content) + end - replace_link_nodes_with_href(ExternalIssue.reference_pattern) do |link, text| - issue_link_filter(link, link_text: text) + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_start_pattern + replace_link_node_with_href(node, link) do + issue_link_filter(link, link_text: text) + end + end + end + end end + + doc end # Replace `JIRA-123` issue references in text with links to the referenced @@ -76,6 +90,21 @@ module Banzai def url_for_issue(*args) IssuesHelper.url_for_issue(*args) end + + def default_issues_tracker? + if RequestStore.active? + default_issues_tracker_cache[project.id] ||= + project.default_issues_tracker? + else + project.default_issues_tracker? + end + end + + private + + def default_issues_tracker_cache + RequestStore[:banzai_default_issues_tracker_cache] ||= {} + end end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 8d368f3b9e7..d179bea181e 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML Filter to add a `rel="nofollow"` attribute to external links diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f31f921903b..d08267a9d6c 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -1,6 +1,3 @@ -require 'banzai' -require 'html/pipeline/filter' - module Banzai module Filter # HTML Filter for parsing Gollum's tags in HTML. It's only parses the @@ -121,7 +118,7 @@ module Banzai end if path - content_tag(:img, nil, src: path) + content_tag(:img, nil, src: path, class: 'gfm') end end @@ -147,12 +144,18 @@ module Banzai # if it is not. def process_page_link_tag(parts) if parts.size == 1 - url = parts[0].strip + reference = parts[0].strip + else + name, reference = *parts.compact.map(&:strip) + end + + if url?(reference) + href = reference else - name, url = *parts.compact.map(&:strip) + href = ::File.join(project_wiki_base_path, reference) end - content_tag(:a, name || url, href: url) + content_tag(:a, name || reference, href: href, class: 'gfm') end def project_wiki diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb new file mode 100644 index 00000000000..ccd106860bd --- /dev/null +++ b/lib/banzai/filter/image_link_filter.rb @@ -0,0 +1,27 @@ +module Banzai + module Filter + # HTML filter that wraps links around inline images. + class ImageLinkFilter < HTML::Pipeline::Filter + + # Find every image that isn't already wrapped in an `a` tag, create + # a new node (a link to the image source), copy the image as a child + # of the anchor, and then replace the img with the link-wrapped version. + def call + doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| + + link = doc.document.create_element( + 'a', + class: 'no-attachment-icon', + href: img['src'], + target: '_blank' + ) + + link.children = img.clone + img.replace(link) + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8147e5ed3c7..a2987850d03 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -31,7 +31,7 @@ module Banzai end def url_for_object(label, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_issues_url(project.namespace, project, label_name: label.name, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 0659fed1419..9b209533a89 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 57c71708992..cad38a51851 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -14,7 +14,7 @@ module Banzai end def url_for_object(mr, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index e88b27c1fae..4cb82178024 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces milestone references with links. @@ -13,7 +11,7 @@ module Banzai end def url_for_object(issue, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 7141ed7c9bd..e589b5df6ec 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that removes references to records that the current user does diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 132f0a4bd93..31386cf851c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,6 +1,3 @@ -require 'active_support/core_ext/string/output_safety' -require 'html/pipeline/filter' - module Banzai module Filter # Base class for GitLab Flavored Markdown reference filters. @@ -55,18 +52,13 @@ module Banzai html.html_safe? ? html : ERB::Util.html_escape_once(html) end - def ignore_parents - @ignore_parents ||= begin - # Don't look for references in text nodes that are children of these - # elements. + def ignore_ancestor_query + @ignore_ancestor_query ||= begin parents = %w(pre code a style) parents << 'blockquote' if context[:ignore_blockquotes] - parents.to_set - end - end - def ignored_ancestry?(node) - has_ancestor?(node, ignore_parents) + parents.map { |n| "ancestor::#{n}" }.join(' or ') + end end def project @@ -77,119 +69,66 @@ module Banzai "gfm gfm-#{type}" end - # Iterate through the document's text nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` AND - # * The node is not an ancestor of an ignored node type - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node's existing content and update the current document. + # Ensure that a :project key exists in context # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_text_nodes_matching(pattern) - return doc if project.nil? - - search_text_nodes(doc).each do |node| - next if ignored_ancestry?(node) - next unless node.text =~ pattern - - content = node.to_html - - html = yield content - - next if html == content - - node.replace(html) - end - - doc + # Note that while the key might exist, its value could be nil! + def validate + needs :project end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node and update the current document. + # Iterates over all <a> and text() nodes in a document. # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_text(pattern) - return doc if project.nil? + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + query = %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] + | descendant-or-self::a[ + not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") + ]} - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') - - link = node.attr('href') - text = node.text - - next unless link && text - - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - # Ignore ending punctionation like periods or commas - next unless link == text && text =~ /\A#{pattern}/ - - html = yield text + doc.xpath(query).each do |node| + yield node + end + end - next if html == text + # Yields the link's URL and text whenever the node is a valid <a> tag. + def yield_valid_link(node) + link = CGI.unescape(node.attr('href').to_s) + text = node.text - node.replace(html) - end + return unless link.force_encoding('UTF-8').valid_encoding? - doc + yield link, text end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's HREF matches `pattern` - # - # pattern - Regex pattern against which to match the node's HREF - # - # Yields the current node's String HREF and String content. - # The result of the block will replace the node and update the current document. - # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_href(pattern) - return doc if project.nil? + def replace_text_when_pattern_matches(node, pattern) + return unless node.text =~ pattern - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') + content = node.to_html + html = yield content - link = node.attr('href') - text = node.text + node.replace(html) unless content == html + end - next unless link && text - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - next unless link && link =~ /\A#{pattern}\z/ + def replace_link_node_with_text(node, link) + html = yield - html = yield link, text + node.replace(html) unless html == node.text + end - next if html == link + def replace_link_node_with_href(node, link) + html = yield - node.replace(html) - end + node.replace(html) unless html == link + end - doc + def text_node?(node) + node.is_a?(Nokogiri::XML::Text) end - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project + def element_node?(node) + node.is_a?(Nokogiri::XML::Element) end end end diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 86d484feb90..96fdb06304e 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that gathers all referenced records that the current user has diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 41380627d39..ea21c7b041c 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index e8011519608..42dbab9d27e 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'html/pipeline/sanitization_filter' - module Banzai module Filter # Sanitize HTML diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index c870a42f741..d507eb5ebe1 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -14,7 +14,7 @@ module Banzai end def url_for_object(snippet, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 8c5855e5ffc..62a79c62e20 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' module Banzai diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 4056dcd6d64..a4eda6fdf76 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that adds an anchor child element to all Headers in a diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index f642aee0967..7edfe5ade2d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 24f16f8b547..eea3af842b6 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -59,13 +59,28 @@ module Banzai end def call - replace_text_nodes_matching(User.reference_pattern) do |content| - user_link_filter(content) + return doc if project.nil? + + ref_pattern = User.reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + each_node do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + user_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_pattern_start + replace_link_node_with_href(node, link) do + user_link_filter(link, link_text: text) + end + end + end + end end - replace_link_nodes_with_href(User.reference_pattern) do |link, text| - user_link_filter(link, link_text: text) - end + doc end # Replace `@user` user references in text with links to the referenced @@ -90,7 +105,7 @@ module Banzai private def urls - Gitlab::Application.routes.url_helpers + Gitlab::Routing.url_helpers end def link_class diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb new file mode 100644 index 00000000000..06d10c98501 --- /dev/null +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -0,0 +1,56 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that "fixes" relative links to files in a repository. + # + # Context options: + # :project_wiki + class WikiLinkFilter < HTML::Pipeline::Filter + + def call + return doc unless project_wiki? + + doc.search('a:not(.gfm)').each do |el| + process_link_attr el.attribute('href') + end + + doc + end + + protected + + def project_wiki? + !context[:project_wiki].nil? + end + + def process_link_attr(html_attr) + return if html_attr.blank? || file_reference?(html_attr) + + uri = URI(html_attr.value) + if uri.relative? && uri.path.present? + html_attr.value = rebuild_wiki_uri(uri).to_s + end + rescue URI::Error + # noop + end + + def rebuild_wiki_uri(uri) + uri.path = ::File.join(project_wiki_base_path, uri.path) + uri + end + + def file_reference?(html_attr) + !File.extname(html_attr.value).blank? + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + end + end +end diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb index e4e2f3f228d..58e3e81209e 100644 --- a/lib/banzai/filter/yaml_front_matter_filter.rb +++ b/lib/banzai/filter/yaml_front_matter_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'yaml' - module Banzai module Filter class YamlFrontMatterFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index f60966c3c0f..321fd5bbe14 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,5 +1,3 @@ -require 'html/pipeline' - module Banzai module Pipeline class BasePipeline diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8cd4b50e65a..ed3cfd6b023 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::UploadLinkFilter, + Filter::ImageLinkFilter, Filter::EmojiFilter, Filter::TableOfContentsFilter, Filter::AutolinkFilter, diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 9b4ff0f0f80..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -1,11 +1,11 @@ -require 'banzai' - module Banzai module Pipeline class WikiPipeline < FullPipeline def self.filters - @filters ||= super.insert_after(Filter::TableOfContentsFilter, - Filter::GollumTagsFilter) + @filters ||= begin + super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) + .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ae714c87dc5..c14a9c4c722 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -19,8 +19,10 @@ module Banzai cache_key = full_cache_key(cache_key, context[:pipeline]) if cache_key - Rails.cache.fetch(cache_key) do - cacheless_render(text, context) + Gitlab::Metrics.measure(:banzai_cached_render) do + Rails.cache.fetch(cache_key) do + cacheless_render(text, context) + end end else cacheless_render(text, context) @@ -64,13 +66,15 @@ module Banzai private def self.cacheless_render(text, context = {}) - result = render_result(text, context) + Gitlab::Metrics.measure(:banzai_cacheless_render) do + result = render_result(text, context) - output = result[:output] - if output.respond_to?(:to_html) - output.to_html - else - output.to_s + output = result[:output] + if output.respond_to?(:to_html) + output.to_html + else + output.to_s + end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 2228425076b..b7209c14148 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -242,9 +242,9 @@ module Ci stage_index = stages.index(job[:stage]) job[:dependencies].each do |dependency| - raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency] + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless stages.index(@jobs[dependency][:stage]) < stage_index + unless stages.index(@jobs[dependency.to_sym][:stage]) < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 6108697bc20..7479e729db1 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,4 +1,7 @@ require 'gitlab/git' module Gitlab + def self.com? + Gitlab.config.gitlab.url == 'https://gitlab.com' + end end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index b9bb6e76081..5e2fb863a8f 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -54,19 +54,6 @@ module Gitlab "#{path}.git", "#{new_path}.git"]) end - # Update HEAD for repository - # - # path - project path with namespace - # branch - repository branch name - # - # Ex. - # update_repository_head("gitlab/gitlab-ci", "3-1-stable") - # - def update_repository_head(path, branch) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'update-head', - "#{path}.git", branch]) - end - # Fork repository to new namespace # # path - project path with namespace @@ -92,33 +79,6 @@ module Gitlab 'rm-project', "#{name}.git"]) end - # Add repository branch from passed ref - # - # path - project path with namespace - # branch_name - new branch name - # ref - HEAD for new branch - # - # Ex. - # add_branch("gitlab/gitlab-ci", "4-0-stable", "master") - # - def add_branch(path, branch_name, ref) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'create-branch', - "#{path}.git", branch_name, ref]) - end - - # Remove repository branch - # - # path - project path with namespace - # branch_name - branch name to remove - # - # Ex. - # rm_branch("gitlab/gitlab-ci", "4-0-stable") - # - def rm_branch(path, branch_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-branch', - "#{path}.git", branch_name]) - end - # Add repository tag from passed ref # # path - project path with namespace @@ -137,19 +97,6 @@ module Gitlab Gitlab::Utils.system_silent(cmd) end - # Remove repository tag - # - # path - project path with namespace - # tag_name - tag name to remove - # - # Ex. - # rm_tag("gitlab/gitlab-ci", "v4.0") - # - def rm_tag(path, tag_name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'rm-tag', - "#{path}.git", tag_name]) - end - # Gc repository # # path - project path with namespace diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb new file mode 100644 index 00000000000..e5e9fab3f5c --- /dev/null +++ b/lib/gitlab/badge/build.rb @@ -0,0 +1,46 @@ +module Gitlab + module Badge + ## + # Build badge + # + class Build + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(project, ref) + @project, @ref = project, ref + @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) + end + + def type + 'image/svg+xml' + end + + def data + File.read(@image[:path]) + end + + def to_s + @image[:name].sub(/\.svg$/, '') + end + + def to_html + link_to(image_tag(image_url, alt: 'build status'), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def image_url + build_namespace_project_badges_url(@project.namespace, + @project, @ref, format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index d88a6eaac6b..9bb507b5edd 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -5,6 +5,17 @@ module Gitlab attr_reader :consumer, :api + def self.from_project(project) + import_data_credentials = project.import_data.credentials if project.import_data + if import_data_credentials && import_data_credentials[:bb_session] + token = import_data_credentials[:bb_session][:bitbucket_access_token] + token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] + new(token, token_secret) + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end + end + def initialize(access_token = nil, access_token_secret = nil) @consumer = ::OAuth::Consumer.new( config.app_id, @@ -54,7 +65,7 @@ module Gitlab def issues(project_identifier) all_issues = [] offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket + per_page = 50 # Maximum number allowed by Bitbucket index = 0 begin @@ -120,7 +131,7 @@ module Gitlab end def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} + Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } end def bitbucket_options diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 46e51a4bf6d..7beaecd1cf0 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -5,10 +5,7 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb index f4dd393ad29..e03c3155b3e 100644 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ b/lib/gitlab/bitbucket_import/key_deleter.rb @@ -6,10 +6,7 @@ module Gitlab def initialize(project) @project = project @current_user = project.creator - import_data = project.import_data.try(:data) - bb_session = import_data["bb_session"] if import_data - @client = Client.new(bb_session["bitbucket_access_token"], - bb_session["bitbucket_access_token_secret"]) + @client = Client.from_project(@project) end def execute diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 03aac1a025a..941f818b847 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -23,7 +23,8 @@ module Gitlab import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", ).execute - project.create_import_data(data: { "bb_session" => session_data } ) + project.create_or_update_import_data(credentials: { bb_session: session_data }) + project end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 761b63e98f6..f44d1b3a44e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -21,7 +21,6 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], - twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], sign_in_text: Settings.extra['sign_in_text'], restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], @@ -35,7 +34,8 @@ module Gitlab max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, two_factor_grace_period: 48, - akismet_enabled: false + akismet_enabled: false, + repository_checks_enabled: true, ) end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 41f0edcaf7e..8f9be6cd9a3 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -5,7 +5,7 @@ module Gitlab attr_accessor :recipient attr_reader :author_id, :ref, :action - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2ca21af5bc8..97ef9851d71 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -45,12 +45,12 @@ module Gitlab note = create_note(reply) unless note.persisted? - message = "The comment could not be created for the following reasons:" + msg = "The comment could not be created for the following reasons:" note.errors.full_messages.each do |error| - message << "\n\n- #{error}" + msg << "\n\n- #{error}" end - raise InvalidNoteError, message + raise InvalidNoteError, msg end end @@ -63,9 +63,24 @@ module Gitlab end def reply_key - reply_key = nil + key_from_to_header || key_from_additional_headers + end + + def key_from_to_header + key = nil message.to.each do |address| - reply_key = Gitlab::IncomingEmail.key_from_address(address) + key = Gitlab::IncomingEmail.key_from_address(address) + break if key + end + + key + end + + def key_from_additional_headers + reply_key = nil + + Array(message.references).each do |message_id| + reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id) break if reply_key end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 2ef50286b1d..ffe49364379 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -15,6 +15,25 @@ module Gitlab # seconds then two overlapping operations may hold a lease for the same # key at the same time. # + # This class has no 'cancel' method. I originally decided against adding + # it because it would add complexity and a false sense of security. The + # complexity: instead of setting '1' we would have to set a UUID, and to + # delete it we would have to execute Lua on the Redis server to only + # delete the key if the value was our own UUID. Otherwise there is a + # chance that when you intend to cancel your lease you actually delete + # someone else's. The false sense of security: you cannot design your + # system to rely too much on the lease being cancelled after use because + # the calling (Ruby) process may crash or be killed. You _cannot_ count + # on begin/ensure blocks to cancel a lease, because the 'ensure' does + # not always run. Think of 'kill -9' from the Unicorn master for + # instance. + # + # If you find that leases are getting in your way, ask yourself: would + # it be enough to lower the lease timeout? Another thing that might be + # appropriate is to only use a lease for bulk/automated operations, and + # to ignore the lease when you get a single 'manual' user request (a + # button click). + # class ExclusiveLease def initialize(key, timeout:) @key, @timeout = key, timeout @@ -24,15 +43,14 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + Gitlab::Redis.with do |redis| + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end end - private + # No #cancel method. See comments above! - def redis - # Maybe someday we want to use a connection pool... - @redis ||= Redis.new(url: Gitlab::RedisConfig.url) - end + private def redis_key "gitlab:exclusive_lease:#{@key}" diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 431d50882fd..2152182b37f 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -26,7 +26,7 @@ module Gitlab def user_map users = {} res = @api.command(:listPeople) - res['people']['person'].each do |user| + [res['people']['person']].flatten.each do |user| users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] } end users diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index db580b5e578..501d5a95547 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -8,17 +8,17 @@ module Gitlab import_data = project.import_data.try(:data) repo_data = import_data['repo'] if import_data - @repo = FogbugzImport::Repository.new(repo_data) - - @known_labels = Set.new + if repo_data + @repo = FogbugzImport::Repository.new(repo_data) + @known_labels = Set.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute return true unless repo.valid? - - data = project.import_data.try(:data) - - client = Gitlab::FogbugzImport::Client.new(token: data['fb_session']['token'], uri: data['fb_session']['uri']) + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @categories = client.categories @@ -30,6 +30,10 @@ module Gitlab private + def fb_session + @import_data_credentials ||= project.import_data.credentials[:fb_session] if project.import_data && project.import_data.credentials + end + def user_map @user_map ||= begin user_map = Hash.new @@ -236,9 +240,8 @@ module Gitlab end def build_attachment_url(rel_url) - data = project.import_data.try(:data) - uri = data['fb_session']['uri'] - token = data['fb_session']['token'] + uri = fb_session[:uri] + token = fb_session[:token] "#{uri}/#{rel_url}&token=#{token}" end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index e0163499e30..3840765db87 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -24,13 +24,7 @@ module Gitlab import_url: Project::UNKNOWN_IMPORT_URL ).execute - project.create_import_data( - data: { - 'repo' => repo.raw_data, - 'user_map' => user_map, - 'fb_session' => fb_session - } - ) + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }, credentials: { fb_session: fb_session }) project end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index a1c6ee7bd69..78d7a4f27cf 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -34,16 +34,21 @@ module Gitlab @source_project = source_project @current_user = current_user @original_html = markdown(text) + @pattern = Gitlab::ReferenceExtractor.references_pattern end def rewrite(target_project) - pattern = Gitlab::ReferenceExtractor.references_pattern + return @text unless needs_rewrite? - @text.gsub(pattern) do |reference| + @text.gsub(@pattern) do |reference| unfold_reference(reference, Regexp.last_match, target_project) end end + def needs_rewrite? + @text =~ @pattern + end + private def unfold_reference(reference, match, target_project) diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb new file mode 100644 index 00000000000..abc8c8c55e6 --- /dev/null +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -0,0 +1,51 @@ +module Gitlab + module Gfm + ## + # Class that rewrites markdown links for uploads + # + # Using a pattern defined in `FileUploader` it copies files to a new + # project and rewrites all links to uploads in in a given text. + # + # + class UploadsRewriter + def initialize(text, source_project, _current_user) + @text = text + @source_project = source_project + @pattern = FileUploader::MARKDOWN_PATTERN + end + + def rewrite(target_project) + return @text unless needs_rewrite? + + @text.gsub(@pattern) do |markdown| + file = find_file(@source_project, $~[:secret], $~[:file]) + return markdown unless file.try(:exists?) + + new_uploader = FileUploader.new(target_project) + new_uploader.store!(file) + new_uploader.to_markdown + end + end + + def needs_rewrite? + files.any? + end + + def files + referenced_files = @text.scan(@pattern).map do + find_file(@source_project, $~[:secret], $~[:file]) + end + + referenced_files.compact.select(&:exists?) + end + + private + + def find_file(project, secret, file) + uploader = FileUploader.new(project, secret) + uploader.retrieve_from_store!(file) + uploader.file + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 172c5441e36..0b1ed510229 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -7,10 +7,12 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - github_session = import_data["github_session"] if import_data - @client = Client.new(github_session["github_access_token"]) - @formatter = Gitlab::ImportFormatter.new + if import_data_credentials + @client = Client.new(import_data_credentials[:user]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute @@ -19,6 +21,10 @@ module Gitlab private + def import_data_credentials + @import_data_credentials ||= project.import_data.credentials if project.import_data + end + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 474927069a5..f4221003db5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, @@ -23,9 +23,6 @@ module Gitlab import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute - - project.create_import_data(data: { "github_session" => session_data } ) - project end end end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 850b73244c6..96717b42bae 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,10 +5,13 @@ module Gitlab def initialize(project) @project = project - import_data = project.import_data.try(:data) - gitlab_session = import_data["gitlab_session"] if import_data - @client = Client.new(gitlab_session["gitlab_access_token"]) - @formatter = Gitlab::ImportFormatter.new + credentials = import_data + if credentials && credentials[:password] + @client = Client.new(credentials[:password]) + @formatter = Gitlab::ImportFormatter.new + else + raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + end end def execute diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 7baaadb813c..77c33db4b59 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -23,7 +23,6 @@ module Gitlab import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - project.create_import_data(data: { "gitlab_session" => session_data } ) project end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb new file mode 100644 index 00000000000..5ebaad6ca6e --- /dev/null +++ b/lib/gitlab/gon_helper.rb @@ -0,0 +1,17 @@ +module Gitlab + module GonHelper + def add_gon_variables + gon.api_version = API::API.version + gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_issues_tracker = Project.new.default_issue_tracker.to_param + gon.max_file_size = current_application_settings.max_attachment_size + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + + if current_user + gon.current_user_id = current_user.id + gon.api_token = current_user.private_token + end + end + end +end diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 87821c23460..0abb7a64c17 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -24,12 +24,7 @@ module Gitlab import_url: repo.import_url ).execute - project.create_import_data( - data: { - "repo" => repo.raw_data, - "user_map" => user_map - } - ) + project.create_or_update_import_data(data: { 'repo' => repo.raw_data, 'user_map' => user_map }) project end diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/import_url.rb new file mode 100644 index 00000000000..d23b013c1f5 --- /dev/null +++ b/lib/gitlab/import_url.rb @@ -0,0 +1,41 @@ +module Gitlab + class ImportUrl + def initialize(url, credentials: nil) + @url = URI.parse(URI.encode(url)) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 9068d79c95e..8ce9d32abe0 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,13 +1,10 @@ module Gitlab module IncomingEmail class << self - def enabled? - config.enabled && address_formatted_correctly? - end + FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze - def address_formatted_correctly? - config.address && - config.address.include?("%{key}") + def enabled? + config.enabled && config.address end def reply_address(key) @@ -24,6 +21,13 @@ module Gitlab match[1] end + def key_from_fallback_reply_message_id(message_id) + match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX) + return unless match + + match[1] + end + def config Gitlab.config.incoming_email end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index da4435c7308..f2b649e50a2 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -33,7 +33,10 @@ module Gitlab def allowed? if ldap_user - return true unless ldap_config.active_directory + unless ldap_config.active_directory + user.activate if user.ldap_blocked? + return true + end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 88a265c6af2..484970c5a10 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -70,6 +70,50 @@ module Gitlab value.to_s.gsub('=', '\\=') end + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def self.measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def self.tag_transaction(name, value) + trans = current_transaction + + trans.add_tag(name, value) if trans + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -81,5 +125,11 @@ module Gitlab new(udp: { host: host, port: port }) end end + + private + + def self.current_transaction + Transaction.current + end end end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 7ea9555cc8c..1cd1ca30f70 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric + JITTER_RANGE = 0.000001..0.001 + attr_reader :series, :values, :tags, :created_at # series - The name of the series (as a String) to store the metric in. @@ -16,11 +18,29 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash + # InfluxDB overwrites an existing point if a new point has the same + # series, tag set, and timestamp. In a highly concurrent environment + # this means that using the number of seconds since the Unix epoch is + # inevitably going to collide with another timestamp. For example, two + # Rails requests processed by different processes may end up generating + # metrics using the _exact_ same timestamp (in seconds). + # + # Due to the way InfluxDB is set up there's no solution to this problem, + # all we can do is lower the amount of collisions. We do this by using + # Time#to_f which returns the seconds as a Float providing greater + # accuracy. We then add a small random value that is large enough to + # distinguish most timestamps but small enough to not alter the amount + # of seconds. + # + # See https://gitlab.com/gitlab-com/operations/issues/175 for more + # information. + time = @created_at.to_f + rand(JITTER_RANGE) + { series: @series, tags: @tags, values: @values, - timestamp: @created_at.to_i * 1_000_000_000 + timestamp: (time * 1_000_000_000).to_i } end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb new file mode 100644 index 00000000000..49e5f86e6e6 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -0,0 +1,39 @@ +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in Rails cache calls + class RailsCache < ActiveSupport::Subscriber + attach_to :active_support + + def cache_read(event) + increment(:cache_read_duration, event.duration) + end + + def cache_write(event) + increment(:cache_write_duration, event.duration) + end + + def cache_delete(event) + increment(:cache_delete_duration, event.duration) + end + + def cache_exist?(event) + increment(:cache_exists_duration, event.duration) + end + + def increment(key, duration) + return unless current_transaction + + current_transaction.increment(:cache_duration, duration) + current_transaction.increment(key, duration) + end + + private + + def current_transaction + Transaction.current + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 83371265278..a7d183b2f94 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -30,6 +30,17 @@ module Gitlab 0 end end + + # THREAD_CPUTIME is not supported on OS X + if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) + def self.cpu_time + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + end + else + def self.cpu_time + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + end + end end end end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index 71cf6a0d886..8bdc89a7751 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -41,7 +41,7 @@ module Gitlab data[:issue] = note.noteable.hook_attrs elsif note.for_merge_request? data[:merge_request] = note.noteable.hook_attrs - elsif note.for_project_snippet? + elsif note.for_snippet? data[:snippet] = note.noteable.hook_attrs end @@ -59,8 +59,7 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } - base_data[:object_attributes][:url] = - Gitlab::UrlBuilder.new(:note).build(note.id) + base_data[:object_attributes][:url] = Gitlab::UrlBuilder.build(note) base_data end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 832fb08a526..356e96fcbab 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -54,6 +54,12 @@ module Gitlab @user ||= build_new_user end + if external_provider? && @user + @user.external = true + elsif @user + @user.external = false + end + @user end @@ -113,6 +119,10 @@ module Gitlab end end + def external_provider? + Gitlab.config.omniauth.external_providers.include?(auth_hash.provider) + end + def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..5c352c96de5 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,50 @@ +module Gitlab + class Redis + CACHE_NAMESPACE = 'cache:gitlab' + SESSION_NAMESPACE = 'session:gitlab' + SIDEKIQ_NAMESPACE = 'resque:gitlab' + + attr_reader :url + + # To be thread-safe we must be careful when writing the class instance + # variables @url and @pool. Because @pool depends on @url we need two + # mutexes to prevent deadlock. + URL_MUTEX = Mutex.new + POOL_MUTEX = Mutex.new + private_constant :URL_MUTEX, :POOL_MUTEX + + def self.url + @url || URL_MUTEX.synchronize { @url = new.url } + end + + def self.with + if @pool.nil? + POOL_MUTEX.synchronize do + @pool = ConnectionPool.new { ::Redis.new(url: url) } + end + end + @pool.with { |redis| yield redis } + end + + def self.redis_store_options + url = new.url + redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb deleted file mode 100644 index 4949c6db539..00000000000 --- a/lib/gitlab/redis_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - class RedisConfig - attr_reader :url - - def self.url - new.url - end - - def self.redis_store_options - url = new.url - redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) - # Redis::Store does not handle Unix sockets well, so let's do it for them - redis_uri = URI.parse(url) - if redis_uri.scheme == 'unix' - redis_config_hash[:path] = redis_uri.path - end - redis_config_hash - end - - def initialize(rails_env=nil) - rails_env ||= Rails.env - config_file = File.expand_path('../../../config/resque.yml', __FILE__) - - @url = "redis://localhost:6379" - if File.exists?(config_file) - @url =YAML.load_file(config_file)[rails_env] - end - end - end -end diff --git a/lib/gitlab/repository_check_logger.rb b/lib/gitlab/repository_check_logger.rb new file mode 100644 index 00000000000..485b596ca57 --- /dev/null +++ b/lib/gitlab/repository_check_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class RepositoryCheckLogger < Gitlab::Logger + def self.file_name_noext + 'repocheck' + end + end +end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb new file mode 100644 index 00000000000..5132177de51 --- /dev/null +++ b/lib/gitlab/routing.rb @@ -0,0 +1,13 @@ +module Gitlab + module Routing + # Returns the URL helpers Module. + # + # This method caches the output as Rails' "url_helpers" method creates an + # anonymous module every time it's called. + # + # Returns a Module. + def self.url_helpers + @url_helpers ||= Gitlab::Application.routes.url_helpers + end + end +end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb new file mode 100644 index 00000000000..32c1c9ec5bb --- /dev/null +++ b/lib/gitlab/saml/auth_hash.rb @@ -0,0 +1,19 @@ +module Gitlab + module Saml + class AuthHash < Gitlab::OAuth::AuthHash + + def groups + get_raw(Gitlab::Saml::Config.groups) + end + + private + + def get_raw(key) + # Needs to call `all` because of https://git.io/vVo4u + # otherwise just the first value is returned + auth_hash.extra[:raw_info].all[key] + end + + end + end +end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb new file mode 100644 index 00000000000..0f40c00f547 --- /dev/null +++ b/lib/gitlab/saml/config.rb @@ -0,0 +1,21 @@ +module Gitlab + module Saml + class Config + + class << self + def options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } + end + + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + end + + end + end +end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index b1e30110ef5..dba4bbfc899 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -18,7 +18,7 @@ module Gitlab @user ||= find_or_create_ldap_user end - if auto_link_saml_enabled? + if auto_link_saml_user? @user ||= find_by_email end @@ -26,6 +26,16 @@ module Gitlab @user ||= build_new_user end + if external_users_enabled? && @user + # Check if there is overlap between the user's groups and the external groups + # setting then set user as external or internal. + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + @user.external = false + else + @user.external = true + end + end + @user end @@ -37,11 +47,24 @@ module Gitlab end end + def changed? + return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) + end + protected - def auto_link_saml_enabled? + def auto_link_saml_user? Gitlab.config.omniauth.auto_link_saml_user end + + def external_users_enabled? + !Gitlab::Saml::Config.external_groups.nil? + end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash) + end end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 6f0d02cafd1..f1943222edf 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,55 +1,61 @@ module Gitlab class UrlBuilder - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers include GitlabRoutingHelper + include ActionView::RecordIdentifier - def initialize(type) - @type = type - end + attr_reader :object - def build(id) - case @type - when :issue - build_issue_url(id) - when :merge_request - build_merge_request_url(id) - when :note - build_note_url(id) + def self.build(object) + new(object).url + end + def url + case object + when Commit + commit_url + when Issue + issue_url(object) + when MergeRequest + merge_request_url(object) + when Note + note_url + else + raise NotImplementedError.new("No URL builder defined for #{object.class}") end end private - def build_issue_url(id) - issue = Issue.find(id) - issue_url(issue) + def initialize(object) + @object = object end - def build_merge_request_url(id) - merge_request = MergeRequest.find(id) - merge_request_url(merge_request) + def commit_url(opts = {}) + return '' if object.project.nil? + + namespace_project_commit_url({ + namespace_id: object.project.namespace, + project_id: object.project, + id: object.id + }.merge!(opts)) end - def build_note_url(id) - note = Note.find(id) - if note.for_commit? - namespace_project_commit_url(namespace_id: note.project.namespace, - id: note.commit_id, - project_id: note.project, - anchor: "note_#{note.id}") - elsif note.for_issue? - issue = Issue.find(note.noteable_id) - issue_url(issue, - anchor: "note_#{note.id}") - elsif note.for_merge_request? - merge_request = MergeRequest.find(note.noteable_id) - merge_request_url(merge_request, - anchor: "note_#{note.id}") - elsif note.for_project_snippet? - snippet = Snippet.find(note.noteable_id) - project_snippet_url(snippet, - anchor: "note_#{note.id}") + def note_url + if object.for_commit? + commit_url(id: object.commit_id, anchor: dom_id(object)) + + elsif object.for_issue? + issue = Issue.find(object.noteable_id) + issue_url(issue, anchor: dom_id(object)) + + elsif object.for_merge_request? + merge_request = MergeRequest.find(object.noteable_id) + merge_request_url(merge_request, anchor: dom_id(object)) + + elsif object.for_snippet? + snippet = Snippet.find(object.noteable_id) + project_snippet_url(snippet, anchor: dom_id(object)) end end end diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci deleted file mode 100644 index bf05edfd780..00000000000 --- a/lib/support/nginx/gitlab_ci +++ /dev/null @@ -1,29 +0,0 @@ -# GITLAB CI -server { - listen 80 default_server; # e.g., listen 192.168.1.1:80; - server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com; - - access_log /var/log/nginx/gitlab_ci_access.log; - error_log /var/log/nginx/gitlab_ci_error.log; - - # expose API to fix runners - location /api { - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_redirect off; - proxy_set_header X-Real-IP $remote_addr; - - # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN - resolver 8.8.8.8 8.8.4.4; - proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # redirect all other CI requests - location / { - return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; - } - - # adjust this to match the largest build log your runners might submit, - # set to 0 to disable limit - client_max_body_size 10m; -}
\ No newline at end of file diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 51e746ef923..2214f855200 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -4,18 +4,19 @@ namespace :cache do desc "GitLab | Clear redis cache" task :clear => :environment do - redis = Redis.new(url: Gitlab::RedisConfig.url) - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index cfaf4a129b1..030ee8bafcb 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,19 +1,50 @@ -# This task will generate a standard and Retina sprite of all of the current -# Gemojione Emojis, with the accompanying SCSS map. -# -# It will not appear in `rake -T` output, and the dependent gems are not -# included in the Gemfile by default, because this task will only be needed -# occasionally, such as when new Emojis are added to Gemojione. - -begin - require 'sprite_factory' - require 'rmagick' -rescue LoadError - # noop -end - namespace :gemojione do + desc 'Generates Emoji SHA256 digests' + task digests: :environment do + require 'digest/sha2' + require 'json' + + dir = Gemojione.index.images_path + digests = [] + aliases = Hash.new { |hash, key| hash[key] = [] } + aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + + JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| + aliases[real_name] << alias_name + end + + AwardEmoji.emojis.map do |name, emoji_hash| + fpath = File.join(dir, "#{emoji_hash['unicode']}.png") + digest = Digest::SHA256.file(fpath).hexdigest + + digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + + aliases[name].each do |alias_name| + digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + end + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(digests)) + end + end + + # This task will generate a standard and Retina sprite of all of the current + # Gemojione Emojis, with the accompanying SCSS map. + # + # It will not appear in `rake -T` output, and the dependent gems are not + # included in the Gemfile by default, because this task will only be needed + # occasionally, such as when new Emojis are added to Gemojione. task sprite: :environment do + begin + require 'sprite_factory' + require 'rmagick' + rescue LoadError + # noop + end + check_requirements! SIZE = 20 diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index cb4abe13799..402bb338f27 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -22,7 +22,7 @@ namespace :gitlab do end # Restore backup of GitLab system - desc "GitLab | Restore a previously created backup" + desc 'GitLab | Restore a previously created backup' task restore: :environment do warn_user_is_not_gitlab configure_cron_mode @@ -30,13 +30,31 @@ namespace :gitlab do backup = Backup::Manager.new backup.unpack - Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") - Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") - Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") - Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") - Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts") - Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs") - Rake::Task["gitlab:shell:setup"].invoke + unless backup.skipped?('db') + unless ENV['force'] == 'yes' + warning = warning = <<-MSG.strip_heredoc + Before restoring the database we recommend removing all existing + tables to avoid future upgrade problems. Be aware that if you have + custom tables in the GitLab database these tables and all data will be + removed. + MSG + ask_to_continue + puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.yellow + sleep(5) + end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + $progress.puts 'Cleaning the database ... '.blue + Rake::Task['gitlab:db:drop_tables'].invoke + $progress.puts 'done'.green + Rake::Task['gitlab:backup:db:restore'].invoke + end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') + Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') + Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') + Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') + Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:shell:setup'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 27ed57efe55..effb8eb6001 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -623,7 +623,6 @@ namespace :gitlab do start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_address_formatted_correctly check_imap_authentication if Rails.env.production? @@ -643,20 +642,6 @@ namespace :gitlab do # Checks ######################## - def check_address_formatted_correctly - print "Address formatted correctly? ... " - - if Gitlab::IncomingEmail.address_formatted_correctly? - puts "yes".green - else - puts "no".red - try_fixing_it( - "Make sure that the address in config/gitlab.yml includes the '%{key}' placeholder." - ) - fix_and_rerun - end - end - def check_initd_configured_correctly print "Init.d configured correctly? ... " diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake new file mode 100644 index 00000000000..4921c6e0bcf --- /dev/null +++ b/lib/tasks/gitlab/db.rake @@ -0,0 +1,35 @@ +namespace :gitlab do + namespace :db do + desc 'GitLab | Manually insert schema migration version' + task :mark_migration_complete, [:version] => :environment do |_, args| + unless args[:version] + puts "Must specify a migration version as an argument".red + exit 1 + end + + version = args[:version].to_i + if version == 0 + puts "Version '#{args[:version]}' must be a non-zero integer".red + exit 1 + end + + sql = "INSERT INTO schema_migrations (version) VALUES (#{version})" + begin + ActiveRecord::Base.connection.execute(sql) + puts "Successfully marked '#{version}' as complete".green + rescue ActiveRecord::RecordNotUnique + puts "Migration version '#{version}' is already marked complete".yellow + end + end + + desc 'Drop all tables' + task :drop_tables => :environment do + connection = ActiveRecord::Base.connection + tables = connection.tables + tables.delete 'schema_migrations' + # Truncate schema_migrations to ensure migrations re-run + connection.execute('TRUNCATE schema_migrations') + tables.each { |t| connection.execute("DROP TABLE #{t}") } + end + end +end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 4cbccf2ca89..48baecfd2a2 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -14,7 +14,7 @@ namespace :gitlab do puts "" end - Rake::Task["db:setup"].invoke + Rake::Task["db:reset"].invoke Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke |