diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-01-26 17:21:38 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-01-26 17:21:38 +0800 |
| commit | 4587c78afa5ff5d2bab718bd632264764860d775 (patch) | |
| tree | 0fb7197a3b94a68746edb3caf65970112703420d /lib | |
| parent | a6394540327cd3919e5189a35a21b57800a104fc (diff) | |
| parent | 403cb125f5e2aced8088f24966624519f6e11e29 (diff) | |
| download | gitlab-ce-4587c78afa5ff5d2bab718bd632264764860d775.tar.gz | |
Merge remote-tracking branch 'upstream/master' into fix-git-hooks-when-creating-file
* upstream/master: (1122 commits)
Update CHANGELOG.md for 8.16.2
Display project ID in project settings (!8572)
fixed points from comments to improve code quality
Update CHANGELOG.md for 8.14.8
Statisfy eslint
Add CHANGELOG entry
Fix access to the wiki code via HTTP when repository feature disabled
Display fullscreen button on small screens (!5302)
Prevent removing fields from dropdowns on input elements
fix for all themes
Return struct instead of multiple values
Fix race conditions for AuthorizedProjectsWorker
Add User#nested_groups and User#nested_projects methods
Fix spec failure due to timestamp ordering issue in mySQL
Fixed error with filter keyboard tests
`can?` already includes the `feature_available?` check
Test there is no Merge Request button when MRs are disabled
Ensure the correct Merge Request button is found
Add 409 conflict tests
Add CHANGELOG
...
Diffstat (limited to 'lib')
137 files changed, 4452 insertions, 575 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 9d5adffd8f4..6cf6b501021 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -14,7 +14,11 @@ module API end # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end + rescue_from Grape::Exceptions::Base do |e| error! e.message, e.status, e.headers end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0950c3d2e88..be659fa4a6a 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -129,12 +129,7 @@ module API end end - # Delete all merged branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/repository/branches/delete_merged + desc 'Delete all merged branches' delete ":id/repository/merged_branches" do DeleteMergedBranchesService.new(user_project, current_user).async_execute diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 4bbdf06a49c..b6e6820c3f4 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -78,6 +78,8 @@ module API description: params[:description] ) + render_validation_error!(status) if status.invalid? + begin case params[:state] when 'pending' diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 031759cdcdf..2fefe760d24 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -44,7 +44,6 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :id, type: Integer, desc: 'The project ID' requires :branch_name, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 85360730841..64da7d6b86f 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -38,26 +38,25 @@ module API present key, with: Entities::SSHKey end - # TODO: for 9.0 we should check if params are there with the params block - # grape provides, at this point we'd change behaviour so we can't - # Behaviour now if you don't provide all required params: it renders a - # validation error or two. desc 'Add new deploy key to currently authenticated user' do success Entities::SSHKey end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end post ":id/#{path}" do - attrs = attributes_for_keys [:title, :key] - attrs[:key].strip! if attrs[:key] + params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys.find_by(key: attrs[:key]) + key = user_project.deploy_keys.find_by(key: params[:key]) if key present key, with: Entities::SSHKey break end # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey @@ -65,7 +64,7 @@ module API end # Create a new deploy key - key = DeployKey.new attrs + key = DeployKey.new(declared_params(include_missing: false)) if key.valid? && user_project.deploy_keys << key present key, with: Entities::SSHKey else @@ -105,15 +104,19 @@ module API present key.deploy_key, with: Entities::SSHKey end - desc 'Delete existing deploy key of currently authenticated user' do + desc 'Delete deploy key for a project' do success Key end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end delete ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find(params[:key_id]) - key.destroy + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + if key + key.destroy + else + not_found!('Deploy Key') + end end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d2fadf6a3d0..9f59939e9ae 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -268,6 +268,13 @@ module API end end + class IssuableTimeStats < Grape::Entity + expose :time_estimate + expose :total_time_spent + expose :human_time_estimate + expose :human_total_time_spent + end + class ExternalIssue < Grape::Entity expose :title expose :id @@ -565,6 +572,8 @@ module API expose :repository_storages expose :koding_enabled expose :koding_url + expose :plantuml_enabled + expose :plantuml_url end class Release < Grape::Entity diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e04d2e40fb6..7682d286866 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -156,12 +156,12 @@ module API success Entities::GroupDetail end params do - requires :project_id, type: String, desc: 'The ID of the project' + requires :project_id, type: String, desc: 'The ID or path of the project' end post ":id/projects/:project_id" do authenticated_as_admin! - group = Group.find_by(id: params[:id]) - project = Project.find(params[:project_id]) + group = find_group!(params[:id]) + project = find_project!(params[:project_id]) result = ::Projects::TransferService.new(project, current_user).execute(group) if result diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ee9247ee240..a1d7b323f4f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,6 +1,7 @@ module API module Helpers include Gitlab::Utils + include Helpers::Pagination SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo @@ -85,10 +86,14 @@ module API IssuesFinder.new(current_user, project_id: user_project.id).find(id) end - def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| - add_pagination_headers(data) - end + def find_project_merge_request(id) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_merge_request_with_access(id, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find(id) + authorize! access_level, merge_request + merge_request end def authenticate! @@ -227,7 +232,7 @@ module API end def render_api_error!(message, status) - error!({ 'message' => message }, status) + error!({ 'message' => message }, status, header) end def handle_api_exception(exception) @@ -299,7 +304,7 @@ module API header['X-Sendfile'] = path body else - file FileStreamer.new(path) + path end end @@ -361,38 +366,6 @@ module API @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] end - def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) - end - - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value - - links = [] - - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? - - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? - - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - - request_params[:page] = paginated_data.total_pages - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") - - links.join(', ') - end - def secret_token Gitlab::Shell.secret_token end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb new file mode 100644 index 00000000000..2199eea7e5f --- /dev/null +++ b/lib/api/helpers/pagination.rb @@ -0,0 +1,45 @@ +module API + module Helpers + module Pagination + def paginate(relation) + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end + end + + private + + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index db2d18f935d..d235977fbd8 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -28,6 +28,8 @@ module API protocol = params[:protocol] + actor.update_last_used_at if actor.is_a?(Key) + access = if wiki? Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) @@ -61,6 +63,8 @@ module API status 200 key = Key.find(params[:key_id]) + key.update_last_used_at + token_handler = Gitlab::LfsToken.new(key) { @@ -103,7 +107,9 @@ module API key = Key.find_by(id: params[:key_id]) - unless key + if key + key.update_last_used_at + else return { 'success' => false, 'message' => 'Could not find the given key' } end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 54b97402426..fe016c1ec0a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -5,13 +5,31 @@ module API before { authenticate! } helpers do - # TODO: Remove in 9.0 and switch to IssueFinder-based label filtering - def filter_issues_labels(issues, labels) - issues.includes(:labels).where('labels.title' => labels.split(',')) + def find_issues(args = {}) + args = params.merge(args) + + args.delete(:id) + args[:milestone_title] = args.delete(:milestone) + + match_all_labels = args.delete(:match_all_labels) + labels = args.delete(:labels) + args[:label_name] = labels if match_all_labels + + args[:search] = "#{Issue.reference_prefix}#{args.delete(:iid)}" if args.key?(:iid) + + issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations + + # TODO: Remove in 9.0 pass `label_name: args.delete(:labels)` to IssuesFinder + if !match_all_labels && labels.present? + issues = issues.includes(:labels).where('labels.title' => labels.split(',')) + end + + issues.reorder(args[:order_by] => args[:sort]) end params :issues_params do optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return issues ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', @@ -40,9 +58,7 @@ module API use :issues_params end get do - issues = IssuesFinder.new(current_user, scope: 'all', author_id: current_user.id, state: params[:state]).execute.inc_notes_with_associations - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = issues.reorder(params[:order_by] => params[:sort]) + issues = find_issues(scope: 'authored') present paginate(issues), with: Entities::Issue, current_user: current_user end @@ -61,15 +77,10 @@ module API use :issues_params end get ":id/issues" do - group = find_group!(params.delete(:id)) - - params[:group_id] = group.id - params[:milestone_title] = params.delete(:milestone) - params[:label_name] = params.delete(:labels) + group = find_group!(params[:id]) - issues = IssuesFinder.new(current_user, params).execute + issues = find_issues(group_id: group.id, state: params[:state] || 'opened', match_all_labels: true) - issues = issues.reorder(params[:order_by] => params[:sort]) present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -78,23 +89,21 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects do + include TimeTrackingEndpoints + desc 'Get a list of project issues' do success Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' - optional :iid, type: Integer, desc: 'The IID of the issue' + optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' use :issues_params end get ":id/issues" do - issues = IssuesFinder.new(current_user, - project_id: user_project.id, - state: params[:state], - milestone_title: params[:milestone]).execute.inc_notes_with_associations - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? - issues = issues.reorder(params[:order_by] => params[:sort]) + project = find_project(params[:id]) + + issues = find_issues(project_id: project.id) present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 07435d78468..bc3d69f6904 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -15,10 +15,8 @@ module API end get ":id/merge_requests/:merge_request_id/versions" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) - authorize! :read_merge_request, merge_request present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff end @@ -34,10 +32,8 @@ module API end get ":id/merge_requests/:merge_request_id/versions/:version_id" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) - authorize! :read_merge_request, merge_request present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 5d1fe22f2df..7ffb38e62da 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -10,6 +10,8 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects do + include TimeTrackingEndpoints + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -96,7 +98,7 @@ module API requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' end delete ":id/merge_requests/:merge_request_id" do - merge_request = user_project.merge_requests.find_by(id: params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) authorize!(:destroy_merge_request, merge_request) merge_request.destroy @@ -116,8 +118,8 @@ module API success Entities::MergeRequest end get path do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end @@ -125,8 +127,8 @@ module API success Entities::RepoCommit end get "#{path}/commits" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present merge_request.commits, with: Entities::RepoCommit end @@ -134,8 +136,8 @@ module API success Entities::MergeRequestChanges end get "#{path}/changes" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end @@ -153,8 +155,7 @@ module API :remove_source_branch end put path do - merge_request = user_project.merge_requests.find(params.delete(:merge_request_id)) - authorize! :update_merge_request, merge_request + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? @@ -180,7 +181,7 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end put "#{path}/merge" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) # Merge request can not be merged # because user dont have permissions to push into target branch @@ -216,7 +217,7 @@ module API success Entities::MergeRequest end post "#{path}/cancel_merge_when_build_succeeds" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) @@ -233,10 +234,7 @@ module API use :pagination end get "#{path}/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - + merge_request = find_merge_request_with_access(params[:merge_request_id]) present paginate(merge_request.notes.fresh), with: Entities::MRNote end @@ -248,8 +246,7 @@ module API requires :note, type: String, desc: 'The text of the comment' end post "#{path}/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :create_note, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) opts = { note: params[:note], @@ -273,7 +270,7 @@ module API use :pagination end get "#{path}/closes_issues" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) present paginate(issues), with: issue_entity(user_project), current_user: current_user end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 284e4cf549a..4d2a8f48267 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -70,21 +70,27 @@ module API end post ":id/#{noteables_str}/:noteable_id/notes" do opts = { - note: params[:body], - noteable_type: noteables_str.classify, - noteable_id: params[:noteable_id] + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: params[:noteable_id] } - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if note.valid? - present note, with: Entities::const_get(note.class.name) + if note.valid? + present note, with: Entities::const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end else - not_found!("Note #{note.errors.messages}") + not_found!("Note") end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3be14e8eb76..941f47114a4 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -159,7 +159,7 @@ module API use :sort_params use :pagination end - get "/search/:query" do + get "/search/:query", requirements: { query: /[^\/]+/ } do search_service = Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page]) projects = projects.reorder(params[:order_by] => params[:sort]) @@ -295,13 +295,13 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? - ::Projects::UpdateService.new(user_project, current_user, attrs).execute + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute - if user_project.errors.any? - render_validation_error!(user_project) - else + if result[:status] == :success present user_project, with: Entities::Project, user_can_admin_project: can?(current_user, :admin_project, user_project) + else + render_validation_error!(user_project) end end diff --git a/lib/api/services.rb b/lib/api/services.rb index d11cdce4e18..a0abec49438 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -145,7 +145,7 @@ module API name: :room, type: String, desc: 'Campfire room' - }, + } ], 'custom-issue-tracker' => [ { @@ -534,7 +534,36 @@ module API desc: 'The password of the user' } ] - }.freeze + } + + service_classes = [ + AsanaService, + AssemblaService, + BambooService, + BugzillaService, + BuildkiteService, + BuildsEmailService, + CampfireService, + CustomIssueTrackerService, + DroneCiService, + EmailsOnPushService, + ExternalWikiService, + FlowdockService, + GemnasiumService, + HipchatService, + IrkerService, + JiraService, + KubernetesService, + MattermostSlashCommandsService, + SlackSlashCommandsService, + PipelinesEmailService, + PivotaltrackerService, + PushoverService, + RedmineService, + SlackService, + MattermostService, + TeamcityService, + ].freeze trigger_services = { 'mattermost-slash-commands' => [ @@ -543,6 +572,13 @@ module API type: String, desc: 'The Mattermost token' } + ], + 'slack-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Slack token' + } ] }.freeze @@ -561,6 +597,19 @@ module API services.each do |service_slug, settings| desc "Set #{service_slug} service for project" params do + service_classes.each do |service| + event_names = service.try(:event_names) || [] + event_names.each do |event_name| + services[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end + end + services.freeze + settings.each do |setting| if setting[:required] requires setting[:name], type: setting[:type], desc: setting[:desc] @@ -574,7 +623,7 @@ module API service_params = declared_params(include_missing: false).merge(active: true) if service.update_attributes(service_params) - true + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? else render_api_error!('400 Bad Request', 400) end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 9eb9a105bde..c5eff16a5de 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -93,6 +93,10 @@ module API given koding_enabled: ->(val) { val } do requires :koding_url, type: String, desc: 'The Koding team URL' end + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' + end optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' @@ -114,7 +118,7 @@ module API :shared_runners_enabled, :max_artifacts_size, :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, :akismet_enabled, :admin_notification_email, :sentry_enabled, - :repository_storage, :repository_checks_enabled, :koding_enabled, + :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, :version_check_enabled, :email_author_in_body, :html_emails_enabled, :housekeeping_enabled end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 10749b34004..e11d7537cc9 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -3,8 +3,8 @@ module API before { authenticate! } subscribable_types = { - 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, - 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, 'labels' => proc { |id| find_project_label(id) }, } diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb new file mode 100644 index 00000000000..85b5f7d98b8 --- /dev/null +++ b/lib/api/time_tracking_endpoints.rb @@ -0,0 +1,114 @@ +module API + module TimeTrackingEndpoints + extend ActiveSupport::Concern + + included do + helpers do + def issuable_name + declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + end + + def issuable_key + "#{issuable_name}_id".to_sym + end + + def update_issuable_key + "update_#{issuable_name}".to_sym + end + + def read_issuable_key + "read_#{issuable_name}".to_sym + end + + def load_issuable + @issuable ||= begin + case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end + end + end + + def update_issuable(attrs) + custom_params = declared_params(include_missing: false) + custom_params.merge!(attrs) + + issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + if issuable.valid? + present issuable, with: Entities::IssuableTimeStats + else + render_validation_error!(issuable) + end + end + + def update_service + issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService + end + end + + issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' + issuable_collection_name = issuable_name.pluralize + issuable_key = "#{issuable_name}_id".to_sym + + desc "Set a time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) + end + + desc "Reset the time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: 0) + end + + desc "Add spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do + authorize! update_issuable_key, load_issuable + + update_issuable(spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user: current_user + }) + end + + desc "Reset spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(spend_time: { duration: :reset, user: current_user }) + end + + desc "Show time stats for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do + authorize! read_issuable_key, load_issuable + + present load_issuable, with: Entities::IssuableTimeStats + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ed8f48aa1e3..9bd077263a7 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -5,7 +5,7 @@ module API before { authenticate! } ISSUABLE_TYPES = { - 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'merge_requests' => ->(id) { find_merge_request_with_access(id) }, 'issues' => ->(id) { find_project_issue(id) } } diff --git a/lib/api/users.rb b/lib/api/users.rb index de07fbf59fc..11a7368b4c0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -91,10 +91,11 @@ module API authenticated_as_admin! # Filter out params which are used later - identity_attrs = params.slice(:provider, :extern_uid) - confirm = params.delete(:confirm) + user_params = declared_params(include_missing: false) + identity_attrs = user_params.slice(:provider, :extern_uid) + confirm = user_params.delete(:confirm) - user = User.new(declared_params(include_missing: false)) + user = User.new(user_params.except(:extern_uid, :provider)) user.skip_confirmation! unless confirm if identity_attrs.any? @@ -159,11 +160,7 @@ module API end end - # Delete already handled parameters - user_params.delete(:extern_uid) - user_params.delete(:provider) - - if user.update_attributes(user_params) + if user.update_attributes(user_params.except(:extern_uid, :provider)) present user, with: Entities::UserPublic else render_validation_error!(user) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7e6537e3d9e..cefbfdce3bb 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -2,6 +2,7 @@ module Backup class Manager ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] FOLDERS_TO_BACKUP = %w[repositories db] + FILE_NAME_SUFFIX = '_gitlab_backup.tar' def pack # Make sure there is a connection @@ -14,7 +15,7 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = s[:backup_created_at].strftime('%s_%Y_%m_%d') + '_gitlab_backup.tar' + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", @@ -82,7 +83,7 @@ module Backup removed = 0 Dir.chdir(Gitlab.config.backup.path) do - Dir.glob('*_gitlab_backup.tar').each do |file| + Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -108,41 +109,50 @@ module Backup Dir.chdir(Gitlab.config.backup.path) # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_backup.tar") - puts "no backups found" if file_list.count == 0 + file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") + + if file_list.count == 0 + $progress.puts "No backups found in #{Gitlab.config.backup.path}" + $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + exit 1 + end if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end - tar_file = ENV["BACKUP"].nil? ? file_list.first : file_list.grep(ENV['BACKUP']).first + if ENV['BACKUP'].present? + tar_file = "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + tar_file = file_list.first + end unless File.exist?(tar_file) - puts "The specified backup doesn't exist!" + $progress.puts "The backup file #{tar_file} does not exist!" exit 1 end - $progress.print "Unpacking backup ... " + $progress.print 'Unpacking backup ... ' unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".color(:red) + $progress.puts 'unpacking backup failed'.color(:red) exit 1 else - $progress.puts "done".color(:green) + $progress.puts 'done'.color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".color(:red) - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) - puts " Please switch to the following version and try again:".color(:red) - puts " version: #{settings[:gitlab_version]}".color(:red) - puts - puts "Hint: git checkout v#{settings[:gitlab_version]}" + $progress.puts 'GitLab version mismatch:'.color(:red) + $progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + $progress.puts ' Please switch to the following version and try again:'.color(:red) + $progress.puts " version: #{settings[:gitlab_version]}".color(:red) + $progress.puts + $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 6d04f68c8f9..a3d495a5da0 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -153,7 +153,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object) + data = data_attributes_for(link_content || match, project, object, link: !!link_content) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -172,9 +172,10 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) data_attribute( original: text, + link: link, project: project.id, object_sym => object.id ) diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4d1bc687696..fd6b9704132 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -62,7 +62,7 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) if object.is_a?(ExternalIssue) data_attribute( project: project.id, diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 84bfeac8041..6640168bfa2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -20,10 +20,10 @@ module Banzai # Examples: # # data_attribute(project: 1, issue: 2) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" # # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String def data_attribute(attributes = {}) @@ -31,7 +31,9 @@ module Banzai attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") + attributes.map do |key, value| + %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") + end.join(' ') end def escape_once(html) @@ -51,6 +53,10 @@ module Banzai context[:project] end + def skip_project_check? + context[:skip_project_check] + end + def reference_class(type) "gfm gfm-#{type} has-tooltip" end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 026b81ac175..a447e2b8bff 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -20,17 +20,19 @@ module Banzai code = node.text css_classes = "code highlight" lexer = lexer_for(language) + lang = lexer.tag begin code = format(lex(lexer, code)) - css_classes << " js-syntax-highlight #{lexer.tag}" + css_classes << " js-syntax-highlight #{lang}" rescue + lang = nil # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index f842b1fb779..1aa9355b256 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai end def call - return doc if project.nil? + return doc if project.nil? && !skip_project_check? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ @@ -58,7 +58,7 @@ module Banzai # have `gfm` and `gfm-project_member` class names attached for styling. def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| - if username == 'all' + if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) elsif namespace = namespaces[username] link_to_namespace(namespace, link_content: link_content) || match diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index ac7bbcb0d10..b64a1287d4d 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -35,7 +35,8 @@ module Banzai src: element['src'], width: '400', controls: true, - 'data-setup' => '{}') + 'data-setup' => '{}', + 'data-title' => element['title'] || element['alt']) link = doc.document.create_element( 'a', diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5a1f873496c..ac95a79009b 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,6 +1,12 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline + # These filters convert GitLab Flavored Markdown (GFM) to HTML. + # The handlers defined in app/assets/javascripts/copy_as_gfm.js.es6 + # consequently convert that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a handler in + # app/assets/javascripts/copy_as_gfm.js.es6, in reverse order. + # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ Filter::SyntaxHighlightFilter, diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index f31fb6c3f71..74663556cbb 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -52,9 +52,9 @@ module Banzai end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field) + def cacheless_render_field(object, field, options = {}) text = object.__send__(field) - context = object.banzai_render_context(field) + context = object.banzai_render_context(field).merge(options) cacheless_render(text, context) end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 229050151d3..c10d3616f31 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -105,7 +105,7 @@ module Ci break elsif s.scan(/</) @out << '<' - elsif s.scan(/\n/) + elsif s.scan(/\r?\n/) @out << '<br>' else @out << s.scan(/./m) diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index a6b9beecded..24bb3649a76 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -8,6 +8,16 @@ module Ci rack_response({ 'message' => '404 Not found' }.to_json, 404) end + # Retain 405 error rather than a 500 error for Grape 0.15.0+. + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end + + rescue_from Grape::Exceptions::Base do |e| + error! e.message, e.status, e.headers + end + rescue_from :all do |exception| handle_api_exception(exception) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 142bce82286..8b939663ffd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -16,17 +16,33 @@ module Ci not_found! unless current_runner.active? update_runner_info - build = Ci::RegisterBuildService.new.execute(current_runner) + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return build_not_found! + end - if build - Gitlab::Metrics.add_event(:build_found, - project: build.project.path_with_namespace) + new_update = current_runner.ensure_runner_queue_value - present build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) + result = Ci::RegisterBuildService.new(current_runner).execute + + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) - build_not_found! + present result.build, with: Entities::BuildDetails + else + Gitlab::Metrics.add_event(:build_not_found) + + header 'X-GitLab-Last-Update', new_update + + build_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index fb04a7824b8..63f9f8d7a5a 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -5,8 +5,8 @@ class EmailTemplateInterceptor def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. unless current_application_settings.html_emails_enabled - message.part.delete_if do |part| - part.content_type.try(:start_with?, 'text/html') + message.parts.delete_if do |part| + part.content_type.start_with?('text/html') end end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index fa234284361..0618107e2c3 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,5 +1,6 @@ require 'asciidoctor' require 'asciidoctor/converter/html5' +require "asciidoctor-plantuml" module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -29,6 +30,8 @@ module Gitlab ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + plantuml_setup + html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.post_process(html, context) @@ -36,6 +39,15 @@ module Gitlab html.html_safe end + def self.plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = ApplicationSetting.current.plantuml_url + conf.svg_enable = ApplicationSetting.current.plantuml_enabled + conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.txt_enable = false + end + end + class Html5Converter < Asciidoctor::Converter::Html5Converter extend Asciidoctor::Converter::Config diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 9c391fa92a3..273118135a9 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -30,9 +30,9 @@ module Gitlab return unless @branch_name return unless project.protected_branch?(@branch_name) - if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + elsif Gitlab::Git.blank_ref?(@newrev) return "You are not allowed to delete protected branches from this project." end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index b7b4b91eb51..f7c530c7d9f 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -33,7 +33,6 @@ module Gitlab validates :url, length: { maximum: 255 }, - addressable_url: true, allow_nil: true validates :action, diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index eee9a64120b..38ac6edc9f1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,8 +4,11 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [Status::Build::Stop, Status::Build::Play, - Status::Build::Cancelable, Status::Build::Retryable] + [[Status::Build::Cancelable, + Status::Build::Retryable], + [Status::Build::FailedAllowed, + Status::Build::Play, + Status::Build::Stop]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb new file mode 100644 index 00000000000..807afe24bd5 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Status + module Build + class FailedAllowed < SimpleDelegator + include Status::Extended + + def label + 'failed (allowed to fail)' + end + + def icon + 'icon_status_warning' + end + + def group + 'failed_with_warnings' + end + + def self.matches?(build, user) + build.failed? && build.allow_failure? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb new file mode 100644 index 00000000000..4969a350862 --- /dev/null +++ b/lib/gitlab/ci/status/external/common.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module External + module Common + def has_details? + subject.target_url.present? && + can?(user, :read_commit_status, subject) + end + + def details_path + subject.target_url + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb new file mode 100644 index 00000000000..07b15bd8d97 --- /dev/null +++ b/lib/gitlab/ci/status/external/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module External + class Factory < Status::Factory + def self.common_helpers + Status::External::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index ae9ef895df4..15836c699c7 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -5,41 +5,46 @@ module Gitlab def initialize(subject, user) @subject = subject @user = user + @status = subject.status || HasStatus::DEFAULT_STATUS end def fabricate! - if extended_status - extended_status.new(core_status) - else + if extended_statuses.none? core_status + else + compound_extended_status end end - def self.extended_statuses - [] + def core_status + Gitlab::Ci::Status + .const_get(@status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) end - def self.common_helpers - Module.new + def compound_extended_status + extended_statuses.inject(core_status) do |status, extended| + extended.new(status) + end end - private + def extended_statuses + return @extended_statuses if defined?(@extended_statuses) - def simple_status - @simple_status ||= @subject.status || :created + groups = self.class.extended_statuses.map do |group| + Array(group).find { |status| status.matches?(@subject, @user) } + end + + @extended_statuses = groups.flatten.compact end - def core_status - Gitlab::Ci::Status - .const_get(simple_status.capitalize) - .new(@subject, @user) - .extend(self.class.common_helpers) + def self.extended_statuses + [] end - def extended_status - @extended ||= self.class.extended_statuses.find do |status| - status.matches?(@subject, @user) - end + def self.common_helpers + Module.new end end end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 16dcb326be9..13c8343b12a 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -4,7 +4,7 @@ module Gitlab module Pipeline class Factory < Status::Factory def self.extended_statuses - [Pipeline::SuccessWithWarnings] + [Status::SuccessWarning] end def self.common_helpers diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb deleted file mode 100644 index 24bf8b869e0..00000000000 --- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb +++ /dev/null @@ -1,31 +0,0 @@ -module Gitlab - module Ci - module Status - module Pipeline - class SuccessWithWarnings < SimpleDelegator - include Status::Extended - - def text - 'passed' - end - - def label - 'passed with warnings' - end - - def icon - 'icon_status_warning' - end - - def group - 'success_with_warnings' - end - - def self.matches?(pipeline, user) - pipeline.success? && pipeline.has_warnings? - end - end - end - end - end -end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 689a5dd45bc..4c37f084d07 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -3,6 +3,10 @@ module Gitlab module Status module Stage class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + def self.common_helpers Status::Stage::Common end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb new file mode 100644 index 00000000000..d4cdab6957a --- /dev/null +++ b/lib/gitlab/ci/status/success_warning.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Status + ## + # Extended status used when pipeline or stage passed conditionally. + # This means that failed jobs that are allowed to fail were present. + # + class SuccessWarning < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def group + 'success_with_warnings' + end + + def self.matches?(subject, user) + subject.success? && subject.has_warnings? + end + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9d142f1b82e..4ebd48a3fc7 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,7 +9,9 @@ module Gitlab end def ensure_application_settings! - if connect_to_db? + return fake_application_settings unless connect_to_db? + + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available @@ -20,42 +22,23 @@ module Gitlab settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end - settings || fake_application_settings + settings || in_memory_application_settings end def sidekiq_throttling_enabled? current_application_settings.sidekiq_throttling_enabled? end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting::DEFAULTS) + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + fake_application_settings + end + def fake_application_settings - OpenStruct.new( - default_projects_limit: Settings.gitlab['default_projects_limit'], - default_branch_protection: Settings.gitlab['default_branch_protection'], - signup_enabled: Settings.gitlab['signup_enabled'], - signin_enabled: Settings.gitlab['signin_enabled'], - gravatar_enabled: Settings.gravatar['enabled'], - koding_enabled: false, - sign_in_text: nil, - after_sign_up_text: nil, - help_page_text: nil, - shared_runners_text: nil, - restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - max_attachment_size: Settings.gitlab['max_attachment_size'], - session_expire_delay: Settings.gitlab['session_expire_delay'], - default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], - default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], - shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Settings.artifacts['max_size'], - require_two_factor_authentication: false, - two_factor_grace_period: 48, - akismet_enabled: false, - repository_checks_enabled: true, - container_registry_token_expire_delay: 5, - user_default_external: false, - sidekiq_throttling_enabled: false, - ) + OpenStruct.new(::ApplicationSetting::DEFAULTS) end private diff --git a/lib/gitlab/cycle_analytics/base_event.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 53a148ad703..0d8791d396b 100644 --- a/lib/gitlab/cycle_analytics/base_event.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -1,13 +1,13 @@ module Gitlab module CycleAnalytics - class BaseEvent - include MetricsTables + class BaseEventFetcher + include BaseQuery - attr_reader :stage, :start_time_attrs, :end_time_attrs, :projections, :query + attr_reader :projections, :query, :stage, :order - def initialize(project:, options:) - @query = EventsQuery.new(project: project, options: options) + def initialize(project:, stage:, options:) @project = project + @stage = stage @options = options end @@ -19,10 +19,8 @@ module Gitlab end.compact end - def custom_query(_base_query); end - def order - @order || @start_time_attrs + @order || default_order end private @@ -34,7 +32,17 @@ module Gitlab end def event_result - @event_result ||= @query.execute(self).to_a + @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a + end + + def events_query + diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + + base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc) + end + + def default_order + [@options[:start_time_attrs]].flatten.first end def serialize(_event) diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb new file mode 100644 index 00000000000..d560dca45c8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -0,0 +1,31 @@ +module Gitlab + module CycleAnalytics + module BaseQuery + include MetricsTables + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + private + + def base_query + @base_query ||= stage_query + end + + def stage_query + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). + join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). + where(issue_table[:project_id].eq(@project.id)). + where(issue_table[:deleted_at].eq(nil)). + where(issue_table[:created_at].gteq(@options[:from])) + + # Load merge_requests + query = query.join(mr_table, Arel::Nodes::OuterJoin). + on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). + join(mr_metrics_table). + on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb new file mode 100644 index 00000000000..74bbcdcb3dd --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -0,0 +1,54 @@ +module Gitlab + module CycleAnalytics + class BaseStage + include BaseQuery + + def initialize(project:, options:) + @project = project + @options = options + end + + def events + event_fetcher.fetch + end + + def as_json + AnalyticsStageSerializer.new.represent(self).as_json + end + + def title + name.to_s.capitalize + end + + def median + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + def name + raise NotImplementedError.new("Expected #{self.name} to implement name") + end + + private + + def event_fetcher + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, + stage: name, + options: event_options) + end + + def event_options + @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_event.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index b394a02cc52..5245b9ca8fc 100644 --- a/lib/gitlab/cycle_analytics/review_event.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -1,22 +1,22 @@ module Gitlab module CycleAnalytics - class ReviewEvent < BaseEvent + class CodeEventFetcher < BaseEventFetcher include MergeRequestAllowed def initialize(*args) - @stage = :review - @start_time_attrs = mr_table[:created_at] - @end_time_attrs = mr_metrics_table[:merged_at] @projections = [mr_table[:title], mr_table[:iid], mr_table[:id], mr_table[:created_at], mr_table[:state], mr_table[:author_id]] + @order = mr_table[:created_at] super(*args) end + private + def serialize(event) AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb new file mode 100644 index 00000000000..d1bc2055ba8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class CodeStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_table[:created_at] + end + + def name + :code + end + + def description + "Time until first merge request" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb new file mode 100644 index 00000000000..50e126cf00b --- /dev/null +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module EventFetcher + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/events.rb b/lib/gitlab/cycle_analytics/events.rb deleted file mode 100644 index 2d703d76cbb..00000000000 --- a/lib/gitlab/cycle_analytics/events.rb +++ /dev/null @@ -1,38 +0,0 @@ -module Gitlab - module CycleAnalytics - class Events - def initialize(project:, options:) - @project = project - @options = options - end - - def issue_events - IssueEvent.new(project: @project, options: @options).fetch - end - - def plan_events - PlanEvent.new(project: @project, options: @options).fetch - end - - def code_events - CodeEvent.new(project: @project, options: @options).fetch - end - - def test_events - TestEvent.new(project: @project, options: @options).fetch - end - - def review_events - ReviewEvent.new(project: @project, options: @options).fetch - end - - def staging_events - StagingEvent.new(project: @project, options: @options).fetch - end - - def production_events - ProductionEvent.new(project: @project, options: @options).fetch - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/events_query.rb b/lib/gitlab/cycle_analytics/events_query.rb deleted file mode 100644 index 2418832ccc2..00000000000 --- a/lib/gitlab/cycle_analytics/events_query.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module CycleAnalytics - class EventsQuery - attr_reader :project - - def initialize(project:, options: {}) - @project = project - @from = options[:from] - @branch = options[:branch] - @fetcher = Gitlab::CycleAnalytics::MetricsFetcher.new(project: project, from: @from, branch: @branch) - end - - def execute(stage_class) - @stage_class = stage_class - - ActiveRecord::Base.connection.exec_query(query.to_sql) - end - - private - - def query - base_query = @fetcher.base_query_for(@stage_class.stage) - diff_fn = @fetcher.subtract_datetimes_diff(base_query, @stage_class.start_time_attrs, @stage_class.end_time_attrs) - - @stage_class.custom_query(base_query) - - base_query.project(extract_epoch(diff_fn).as('total_time'), *@stage_class.projections).order(@stage_class.order.desc) - end - - def extract_epoch(arel_attribute) - return arel_attribute unless Gitlab::Database.postgresql? - - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{arel_attribute.to_sql}))}) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_event.rb b/lib/gitlab/cycle_analytics/issue_event.rb deleted file mode 100644 index 705b7e5ce24..00000000000 --- a/lib/gitlab/cycle_analytics/issue_event.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module CycleAnalytics - class IssueEvent < BaseEvent - include IssueAllowed - - def initialize(*args) - @stage = :issue - @start_time_attrs = issue_table[:created_at] - @end_time_attrs = [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(*args) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event).as_json - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_event.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 4868c3c6237..0d8da99455e 100644 --- a/lib/gitlab/cycle_analytics/production_event.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -1,12 +1,9 @@ module Gitlab module CycleAnalytics - class ProductionEvent < BaseEvent + class IssueEventFetcher < BaseEventFetcher include IssueAllowed def initialize(*args) - @stage = :production - @start_time_attrs = issue_table[:created_at] - @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] @projections = [issue_table[:title], issue_table[:iid], issue_table[:id], diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb new file mode 100644 index 00000000000..d2068fbc38f --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class IssueStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def name + :issue + end + + def description + "Time before an issue gets scheduled" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_fetcher.rb b/lib/gitlab/cycle_analytics/metrics_fetcher.rb deleted file mode 100644 index b71e8735e27..00000000000 --- a/lib/gitlab/cycle_analytics/metrics_fetcher.rb +++ /dev/null @@ -1,60 +0,0 @@ -module Gitlab - module CycleAnalytics - class MetricsFetcher - include Gitlab::Database::Median - include Gitlab::Database::DateTime - include MetricsTables - - DEPLOYMENT_METRIC_STAGES = %i[production staging] - - def initialize(project:, from:, branch:) - @project = project - @project = project - @from = from - @branch = branch - end - - def calculate_metric(name, start_time_attrs, end_time_attrs) - cte_table = Arel::Table.new("cte_table_for_#{name}") - - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. - interval_query = Arel::Nodes::As.new( - cte_table, - subtract_datetimes(base_query_for(name), start_time_attrs, end_time_attrs, name.to_s)) - - median_datetime(cte_table, interval_query, name) - end - - # Join table with a row for every <issue,merge_request> pair (where the merge request - # closes the given issue) with issue and merge request metrics included. The metrics - # are loaded with an inner join, so issues / merge requests without metrics are - # automatically excluded. - def base_query_for(name) - # Load issues - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). - join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). - where(issue_table[:project_id].eq(@project.id)). - where(issue_table[:deleted_at].eq(nil)). - where(issue_table[:created_at].gteq(@from)) - - query = query.where(build_table[:ref].eq(@branch)) if name == :test && @branch - - # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin). - on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). - join(mr_metrics_table). - on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - - if DEPLOYMENT_METRIC_STAGES.include?(name) - # Limit to merge requests that have been deployed to production after `@from` - query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) - end - - query - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_event.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7c3f0e9989f..88a8710dbe6 100644 --- a/lib/gitlab/cycle_analytics/plan_event.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -1,19 +1,17 @@ module Gitlab module CycleAnalytics - class PlanEvent < BaseEvent + class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @stage = :plan - @start_time_attrs = issue_metrics_table[:first_associated_with_milestone_at] - @end_time_attrs = [issue_metrics_table[:first_added_to_board_at], - issue_metrics_table[:first_mentioned_in_commit_at]] @projections = [mr_diff_table[:st_commits].as('commits'), issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end - def custom_query(base_query) + def events_query base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + + super end private diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb new file mode 100644 index 00000000000..3b4dfc6a30e --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class PlanStage < BaseStage + def start_time_attrs + @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def end_time_attrs + @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def name + :plan + end + + def description + "Time before an issue starts implementation" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb new file mode 100644 index 00000000000..0fa2e87f673 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class ProductionEventFetcher < IssueEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb new file mode 100644 index 00000000000..d693443bfa4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module ProductionHelper + def stage_query + super.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@options[:from])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb new file mode 100644 index 00000000000..2a6bcc80116 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -0,0 +1,28 @@ +module Gitlab + module CycleAnalytics + class ProductionStage < BaseStage + include ProductionHelper + + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :production + end + + def description + "From issue creation until deploy to production" + end + + def query + # Limit to merge requests that have been deployed to production after `@from` + query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index 2afdf0b8518..4df0bd06393 100644 --- a/lib/gitlab/cycle_analytics/code_event.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -1,25 +1,19 @@ module Gitlab module CycleAnalytics - class CodeEvent < BaseEvent + class ReviewEventFetcher < BaseEventFetcher include MergeRequestAllowed def initialize(*args) - @stage = :code - @start_time_attrs = issue_metrics_table[:first_mentioned_in_commit_at] - @end_time_attrs = mr_table[:created_at] @projections = [mr_table[:title], mr_table[:iid], mr_table[:id], mr_table[:created_at], mr_table[:state], mr_table[:author_id]] - @order = mr_table[:created_at] super(*args) end - private - def serialize(event) AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb new file mode 100644 index 00000000000..fbaa3010d81 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class ReviewStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:merged_at] + end + + def name + :review + end + + def description + "Time between merge request creation and merge/close" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb new file mode 100644 index 00000000000..28e0455df59 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module Stage + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb new file mode 100644 index 00000000000..b34baf5b081 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class StageSummary + def initialize(project, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def data + [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)), + serialize(Summary::Commit.new(project: @project, from: @from)), + serialize(Summary::Deploy.new(project: @project, from: @from))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_event.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index a1f30b716f6..a34731a5fcd 100644 --- a/lib/gitlab/cycle_analytics/staging_event.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -1,10 +1,7 @@ module Gitlab module CycleAnalytics - class StagingEvent < BaseEvent + class StagingEventFetcher < BaseEventFetcher def initialize(*args) - @stage = :staging - @start_time_attrs = mr_metrics_table[:merged_at] - @end_time_attrs = mr_metrics_table[:first_deployed_to_production_at] @projections = [build_table[:id]] @order = build_table[:created_at] @@ -17,8 +14,10 @@ module Gitlab super end - def custom_query(base_query) + def events_query base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super end private diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb new file mode 100644 index 00000000000..945909a4d62 --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class StagingStage < BaseStage + include ProductionHelper + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:merged_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :staging + end + + def description + "From merge request merge until deploy to production" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb new file mode 100644 index 00000000000..43fa3795e5c --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -0,0 +1,20 @@ +module Gitlab + module CycleAnalytics + module Summary + class Base + def initialize(project:, from:) + @project = project + @from = from + end + + def title + self.class.name.demodulize + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb new file mode 100644 index 00000000000..7b8faa4d854 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -0,0 +1,39 @@ +module Gitlab + module CycleAnalytics + module Summary + class Commit < Base + def value + @value ||= count_commits + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + output, status = Gitlab::Popen.popen(cmd) + + raise IOError, output unless status.zero? + + output.lines.count + end + + def ref + @ref ||= @project.default_branch.presence + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb new file mode 100644 index 00000000000..06032e9200e --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -0,0 +1,11 @@ +module Gitlab + module CycleAnalytics + module Summary + class Deploy < Base + def value + @value ||= @project.deployments.where("created_at > ?", @from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb new file mode 100644 index 00000000000..008468f24b9 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + module Summary + class Issue < Base + def initialize(project:, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def title + 'New Issue' + end + + def value + @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event.rb b/lib/gitlab/cycle_analytics/test_event.rb deleted file mode 100644 index d553d0b5aec..00000000000 --- a/lib/gitlab/cycle_analytics/test_event.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Gitlab - module CycleAnalytics - class TestEvent < StagingEvent - def initialize(*args) - super(*args) - - @stage = :test - @start_time_attrs = mr_metrics_table[:latest_build_started_at] - @end_time_attrs = mr_metrics_table[:latest_build_finished_at] - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb new file mode 100644 index 00000000000..a2589c6601a --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class TestEventFetcher < StagingEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb new file mode 100644 index 00000000000..0079d56e0e4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -0,0 +1,29 @@ +module Gitlab + module CycleAnalytics + class TestStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] + end + + def name + :test + end + + def description + "Total test time for all commits/merges" + end + + def stage_query + if @options[:branch] + super.where(build_table[:ref].eq(@options[:branch])) + else + super + end + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 1444d25ebc7..08607c27c09 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -103,6 +103,11 @@ module Gitlab Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) end + def extract_diff_epoch(diff) + return diff unless Gitlab::Database.postgresql? + + Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) + end # Need to cast '0' to an INTERVAL before we can check if the interval is positive def zero_interval Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index bd3267e2a80..bd2f5d3615e 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,10 +1,11 @@ require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' +require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [CreateNoteHandler, CreateIssueHandler] + HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler] def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 7cccf465334..3f6ace0311a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -9,52 +9,13 @@ module Gitlab @mail_key = mail_key end - def message - @message ||= process_message - end - - def author + def can_execute? raise NotImplementedError end - def project + def execute raise NotImplementedError end - - private - - def validate_permission!(permission) - raise UserNotFoundError unless author - raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) - end - - def process_message - message = ReplyParser.new(mail).execute.strip - add_attachments(message) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) - - reply + attachments.map do |link| - "\n\n#{link[:markdown]}" - end.join - end - - def verify_record!(record:, invalid_exception:, record_name:) - return if record.persisted? - return if record.errors.key?(:commands_only) - - error_title = "The #{record_name} could not be created for the following reasons:" - - msg = error_title + record.errors.full_messages.map do |error| - "\n\n- #{error}" - end.join - - raise invalid_exception, msg - end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 9f90a3ec2b2..127fae159d5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,6 +5,7 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler + include ReplyProcessing attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 447c7a6a6b9..d87ba427f4b 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,10 +1,13 @@ require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' module Gitlab module Email module Handler class CreateNoteHandler < BaseHandler + include ReplyProcessing + def can_handle? mail_key =~ /\A\w+\z/ end @@ -24,6 +27,8 @@ module Gitlab record_name: 'comment') end + private + def author sent_notification.recipient end @@ -36,8 +41,6 @@ module Gitlab @sent_notification ||= SentNotification.for(mail_key) end - private - def create_note Notes::CreateService.new( project, diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb new file mode 100644 index 00000000000..32c5caf93e8 --- /dev/null +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -0,0 +1,54 @@ +module Gitlab + module Email + module Handler + module ReplyProcessing + private + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + def message + @message ||= process_message + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb new file mode 100644 index 00000000000..97d7a8d65ff --- /dev/null +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -0,0 +1,32 @@ +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class UnsubscribeHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + return unless sent_notification.unsubscribable? + + noteable = sent_notification.noteable + raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) + end + + private + + def sent_notification + @sent_notification ||= SentNotification.for(reply_key) + end + + def reply_key + mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + end + end + end + end +end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb new file mode 100644 index 00000000000..42140ecc993 --- /dev/null +++ b/lib/gitlab/git/attributes.rb @@ -0,0 +1,131 @@ +module Gitlab + module Git + # Class for parsing Git attribute files and extracting the attributes for + # file patterns. + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::Attributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class Attributes + # path - The path to the Git repository. + def initialize(path) + @path = File.expand_path(path) + @patterns = nil + end + + # Returns all the Git attributes for the given path. + # + # path - A path to a file for which to get the attributes. + # + # Returns a Hash. + def attributes(path) + full_path = File.join(@path, path) + + patterns.each do |pattern, attrs| + return attrs if File.fnmatch?(pattern, full_path) + end + + {} + end + + # Returns a Hash containing the file patterns and their attributes. + def patterns + @patterns ||= parse_file + end + + # Parses an attribute string. + # + # These strings can be in the following formats: + # + # text # => { "text" => true } + # -text # => { "text" => false } + # key=value # => { "key" => "value" } + # + # string - The string to parse. + # + # Returns a Hash containing the attributes and their values. + def parse_attributes(string) + values = {} + dash = '-' + equal = '=' + binary = 'binary' + + string.split(/\s+/).each do |chunk| + # Data such as "foo = bar" should be treated as "foo" and "bar" being + # separate boolean attributes. + next if chunk == equal + + key = chunk + + # Input: "-foo" + if chunk.start_with?(dash) + key = chunk.byteslice(1, chunk.length - 1) + value = false + + # Input: "foo=bar" + elsif chunk.include?(equal) + key, value = chunk.split(equal, 2) + + # Input: "foo" + else + value = true + end + + values[key] = value + + # When the "binary" option is set the "diff" option should be set to + # the inverse. If "diff" is later set it should overwrite the + # automatically set value. + values['diff'] = false if key == binary && value + end + + values + end + + # Iterates over every line in the attributes file. + def each_line + full_path = File.join(@path, 'info/attributes') + + return unless File.exist?(full_path) + + File.open(full_path, 'r') do |handle| + handle.each_line do |line| + break unless line.valid_encoding? + + yield line.strip + end + end + end + + private + + # Parses the Git attributes file. + def parse_file + pairs = [] + comment = '#' + + each_line do |line| + next if line.start_with?(comment) || line.empty? + + pattern, attrs = line.split(/\s+/, 2) + + parsed = attrs ? parse_attributes(attrs) : {} + + pairs << [File.join(@path, pattern), parsed] + end + + # Newer entries take precedence over older entries. + pairs.reverse.to_h + end + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb new file mode 100644 index 00000000000..58193391926 --- /dev/null +++ b/lib/gitlab/git/blame.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class Blame + include Gitlab::Git::EncodingHelper + + attr_reader :lines, :blames + + def initialize(repository, sha, path) + @repo = repository + @sha = sha + @path = path + @lines = [] + @blames = load_blame + end + + def each + @blames.each do |blame| + yield( + Gitlab::Git::Commit.new(blame.commit), + blame.line + ) + end + end + + private + + def load_blame + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + raw_output = IO.popen(cmd, 'rb') {|io| io.read } + output = encode_utf8(raw_output) + process_raw_blame output + end + + def process_raw_blame(output) + lines, final = [], [] + info, commits = {}, {} + + # process the output + output.split("\n").each do |line| + if line[0, 1] == "\t" + lines << line[1, line.size] + elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) + commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commits[commit_id] = nil unless commits.key?(commit_id) + info[lineno] = [commit_id, old_lineno] + end + end + + # load all commits in single call + commits.keys.each do |key| + commits[key] = @repo.lookup(key) + end + + # get it together + info.sort.each do |lineno, (commit_id, old_lineno)| + commit = commits[commit_id] + final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + end + + @lines = final + end + end + + class BlameLine + attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) + @lineno = lineno + @oldlineno = oldlineno + @commit = commit + @line = line + end + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb new file mode 100644 index 00000000000..b742d9e1e4b --- /dev/null +++ b/lib/gitlab/git/blob.rb @@ -0,0 +1,330 @@ +module Gitlab + module Git + class Blob + include Linguist::BlobHelper + include Gitlab::Git::EncodingHelper + + # This number is the maximum amount of data that we want to display to + # the user. We load as much as we can for encoding detection + # (Linguist) and LFS pointer parsing. All other cases where we need full + # blob data should use load_all_data!. + MAX_DATA_DISPLAY_SIZE = 10485760 + + attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary + + class << self + def find(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end + + def raw(repository, sha) + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + # Strip leading slashes + path[/^\/*/] = '' + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] + end + + return nil unless entry + + if path_arr.size > 1 + return nil unless entry[:type] == :tree + path_arr.shift + find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + data: '', + path: path, + commit_id: sha, + ) + end + + # Commit file in repository and return commit sha + # + # options should contain next structure: + # file: { + # content: 'Lorem ipsum...', + # path: 'documents/story.txt', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + # + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def commit(repository, options, action = :add) + file = options[:file] + update = file[:update].nil? ? true : file[:update] + author = options[:author] + committer = options[:committer] + commit = options[:commit] + repo = repository.rugged + ref = commit[:branch] + update_ref = commit[:update_ref].nil? ? true : commit[:update_ref] + parents = [] + mode = 0o100644 + + unless ref.start_with?('refs/') + ref = 'refs/heads/' + ref + end + + path_name = Gitlab::Git::PathHelper.normalize_path(file[:path]) + # Abort if any invalid characters remain (e.g. ../foo) + raise Gitlab::Git::Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..') + + filename = path_name.to_s + index = repo.index + + unless repo.empty? + rugged_ref = repo.references[ref] + raise Gitlab::Git::Repository::InvalidRef.new("Invalid branch name") unless rugged_ref + last_commit = rugged_ref.target + index.read_tree(last_commit.tree) + parents = [last_commit] + end + + if action == :remove + index.remove(filename) + else + file_entry = index.get(filename) + + if action == :rename + old_path_name = Gitlab::Git::PathHelper.normalize_path(file[:previous_path]) + old_filename = old_path_name.to_s + file_entry = index.get(old_filename) + index.remove(old_filename) unless file_entry.blank? + end + + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update + + # Preserve the current file mode if one is available + mode = file_entry[:mode] if file_entry[:mode] + end + + content = file[:content] + detect = CharlockHolmes::EncodingDetector.new.detect(content) if content + + unless detect && detect[:type] == :binary + # When writing to the repo directly as we are doing here, + # the `core.autocrlf` config isn't taken into account. + content.gsub!("\r\n", "\n") if repository.autocrlf + end + + oid = repo.write(content, :blob) + index.add(path: filename, oid: oid, mode: mode) + end + + opts = {} + opts[:tree] = index.write_tree(repo) + opts[:author] = author + opts[:committer] = committer + opts[:message] = commit[:message] + opts[:parents] = parents + opts[:update_ref] = ref if update_ref + + Rugged::Commit.create(repo, opts) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity + + # Remove file from repository and return commit sha + # + # options should contain next structure: + # file: { + # path: 'documents/story.txt' + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Remove FILENAME', + # branch: 'master' + # } + # + def remove(repository, options) + commit(repository, options, :remove) + end + + # Rename file from repository and return commit sha + # + # options should contain next structure: + # file: { + # previous_path: 'documents/old_story.txt' + # path: 'documents/story.txt' + # content: 'Lorem ipsum...', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Rename FILENAME', + # branch: 'master' + # } + # + def rename(repository, options) + commit(repository, options, :rename) + end + end + + def initialize(options) + %w(id name path size data mode commit_id binary).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + + @loaded_all_data = false + # Retain the actual size before it is encoded + @loaded_size = @data.bytesize if @data + end + + def binary? + @binary.nil? ? super : @binary == true + end + + def empty? + !data || data == '' + end + + def data + encode! @data + end + + # Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into + # memory as a Ruby string. + def load_all_data!(repository) + return if @data == '' # don't mess with submodule blobs + return @data if @loaded_all_data + + @loaded_all_data = true + @data = repository.lookup(id).content + @loaded_size = @data.bytesize + end + + def name + encode! @name + end + + # Valid LFS object pointer is a text file consisting of + # version + # oid + # size + # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer + def lfs_pointer? + has_lfs_version_key? && lfs_oid.present? && lfs_size.present? + end + + def lfs_oid + if has_lfs_version_key? + oid = data.match(/(?<=sha256:)([0-9a-f]{64})/) + return oid[1] if oid + end + + nil + end + + def lfs_size + if has_lfs_version_key? + size = data.match(/(?<=size )([0-9]+)/) + return size[1] if size + end + + nil + end + + def truncated? + size && (size > loaded_size) + end + + private + + def has_lfs_version_key? + !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + end + end + end +end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb new file mode 100644 index 00000000000..e98de57fc22 --- /dev/null +++ b/lib/gitlab/git/blob_snippet.rb @@ -0,0 +1,32 @@ +module Gitlab + module Git + class BlobSnippet + include Linguist::BlobHelper + + attr_accessor :ref + attr_accessor :lines + attr_accessor :filename + attr_accessor :startline + + def initialize(ref, lines, startline, filename) + @ref, @lines, @startline, @filename = ref, lines, startline, filename + end + + def data + lines.join("\n") if lines + end + + def name + filename + end + + def size + data.length + end + + def mode + nil + end + end + end +end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb new file mode 100644 index 00000000000..586380da94a --- /dev/null +++ b/lib/gitlab/git/branch.rb @@ -0,0 +1,6 @@ +module Gitlab + module Git + class Branch < Ref + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb new file mode 100644 index 00000000000..d785516ebdd --- /dev/null +++ b/lib/gitlab/git/commit.rb @@ -0,0 +1,310 @@ +# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object +module Gitlab + module Git + class Commit + include Gitlab::Git::EncodingHelper + + attr_accessor :raw_commit, :head, :refs + + SERIALIZE_KEYS = [ + :id, :message, :parent_ids, + :authored_date, :author_name, :author_email, + :committed_date, :committer_name, :committer_email + ].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + def ==(other) + return false unless other.is_a?(Gitlab::Git::Commit) + + methods = [:message, :parent_ids, :authored_date, :author_name, + :author_email, :committed_date, :committer_name, + :committer_email] + + methods.all? do |method| + send(method) == other.send(method) + end + end + + class << self + # Get commits collection + # + # Ex. + # Commit.where( + # repo: repo, + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # ) + # + def where(options) + repo = options.delete(:repo) + raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) + + repo.log(options).map { |c| decorate(c) } + end + + # Get single commit + # + # Ex. + # Commit.find(repo, '29eda46b') + # + # Commit.find(repo, 'master') + # + def find(repo, commit_id = "HEAD") + return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) + + obj = if commit_id.is_a?(String) + repo.rev_parse_target(commit_id) + else + Gitlab::Git::Ref.dereference_object(commit_id) + end + + return nil unless obj.is_a?(Rugged::Commit) + + decorate(obj) + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + nil + end + + # Get last commit for HEAD + # + # Ex. + # Commit.last(repo) + # + def last(repo) + find(repo) + end + + # Get last commit for specified path and ref + # + # Ex. + # Commit.last_for_path(repo, '29eda46b', 'app/models') + # + # Commit.last_for_path(repo, 'master', 'Gemfile') + # + def last_for_path(repo, ref, path = nil) + where( + repo: repo, + ref: ref, + path: path, + limit: 1 + ).first + end + + # Get commits between two revspecs + # See also #repository.commits_between + # + # Ex. + # Commit.between(repo, '29eda46b', 'master') + # + def between(repo, base, head) + repo.commits_between(base, head).map do |commit| + decorate(commit) + end + rescue Rugged::ReferenceError + [] + end + + # Delegate Repository#find_commits + def find_all(repo, options = {}) + repo.find_commits(options) + end + + def decorate(commit, ref = nil) + Gitlab::Git::Commit.new(commit, ref) + end + + # Returns a diff object for the changes introduced by +rugged_commit+. + # If +rugged_commit+ doesn't have a parent, then the diff is between + # this commit and an empty repo. See Repository#diff for the keys + # allowed in the +options+ hash. + def diff_from_parent(rugged_commit, options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff + end + end + + def initialize(raw_commit, head = nil) + raise "Nil as raw commit passed" unless raw_commit + + if raw_commit.is_a?(Hash) + init_from_hash(raw_commit) + elsif raw_commit.is_a?(Rugged::Commit) + init_from_rugged(raw_commit) + else + raise "Invalid raw commit type: #{raw_commit.class}" + end + + @head = head + end + + def sha + id + end + + def short_id(length = 10) + id.to_s[0..length] + end + + def safe_message + @safe_message ||= message + end + + def created_at + committed_date + end + + # Was this commit committed by a different person than the original author? + def different_committer? + author_name != committer_name || author_email != committer_email + end + + def parent_id + parent_ids.first + end + + # Shows the diff between the commit's parent and the commit. + # + # Cuts out the header and stats from #to_patch and returns only the diff. + def to_diff(options = {}) + diff_from_parent(options).patch + end + + # Returns a diff object for the changes from this commit's first parent. + # If there is no parent, then the diff is between this commit and an + # empty repo. See Repository#diff for keys allowed in the +options+ + # hash. + def diff_from_parent(options = {}) + Commit.diff_from_parent(raw_commit, options) + end + + def has_zero_stats? + stats.total.zero? + rescue + true + end + + def no_commit_message + "--no commit message" + end + + def to_hash + serialize_keys.map.with_object({}) do |key, hash| + hash[key] = send(key) + end + end + + def date + committed_date + end + + def diffs(options = {}) + Gitlab::Git::DiffCollection.new(diff_from_parent(options), options) + end + + def parents + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + end + + def tree + raw_commit.tree + end + + def stats + Gitlab::Git::CommitStats.new(self) + end + + def to_patch(options = {}) + begin + raw_commit.to_mbox(options) + rescue Rugged::InvalidError => ex + if ex.message =~ /Commit \w+ is a merge commit/ + 'Patch format is not currently supported for merge commits.' + end + end + end + + # Get a collection of Rugged::Reference objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end + + # Get ref names collection + # + # Ex. + # commit.ref_names(repo) + # + def ref_names(repo) + refs(repo).map do |ref| + ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + end + end + + def message + encode! @message + end + + def author_name + encode! @author_name + end + + def author_email + encode! @author_email + end + + def committer_name + encode! @committer_name + end + + def committer_email + encode! @committer_email + end + + private + + def init_from_hash(hash) + raw_commit = hash.symbolize_keys + + serialize_keys.each do |key| + send("#{key}=", raw_commit[key]) + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + + def serialize_keys + SERIALIZE_KEYS + end + end + end +end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb new file mode 100644 index 00000000000..e9118bbed0e --- /dev/null +++ b/lib/gitlab/git/commit_stats.rb @@ -0,0 +1,26 @@ +# Gitlab::Git::CommitStats counts the additions, deletions, and total changes +# in a commit. +module Gitlab + module Git + class CommitStats + attr_reader :id, :additions, :deletions, :total + + # Instantiate a CommitStats object + def initialize(commit) + @id = commit.id + @additions = 0 + @deletions = 0 + @total = 0 + + diff = commit.diff_from_parent + + diff.each_patch do |p| + # TODO: Use the new Rugged convenience methods when they're released + @additions += p.stat[0] + @deletions += p.stat[1] + @total += p.changes + end + end + end + end +end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb new file mode 100644 index 00000000000..696a2acd5e3 --- /dev/null +++ b/lib/gitlab/git/compare.rb @@ -0,0 +1,43 @@ +module Gitlab + module Git + class Compare + attr_reader :head, :base, :straight + + def initialize(repository, base, head, straight = false) + @repository = repository + @straight = straight + + unless base && head + @commits = [] + return + end + + @base = Gitlab::Git::Commit.find(repository, base.try(:strip)) + @head = Gitlab::Git::Commit.find(repository, head.try(:strip)) + + @commits = [] unless @base && @head + @commits = [] if same + end + + def same + @base && @head && @base.id == @head.id + end + + def commits + return @commits if defined?(@commits) + + @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id) + end + + def diffs(options = {}) + unless @head && @base + return Gitlab::Git::DiffCollection.new([]) + end + + paths = options.delete(:paths) || [] + options[:straight] = @straight + Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb new file mode 100644 index 00000000000..d6b3b5705a9 --- /dev/null +++ b/lib/gitlab/git/diff.rb @@ -0,0 +1,322 @@ +# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object +module Gitlab + module Git + class Diff + class TimeoutError < StandardError; end + include Gitlab::Git::EncodingHelper + + # Diff properties + attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff + + # Stats properties + attr_accessor :new_file, :renamed_file, :deleted_file + + attr_accessor :too_large + + # The maximum size of a diff to display. + DIFF_SIZE_LIMIT = 102400 # 100 KB + + # The maximum size before a diff is collapsed. + DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + + class << self + def between(repo, head, base, options = {}, *paths) + straight = options.delete(:straight) || false + + common_commit = if straight + base + else + # Only show what is new in the source branch + # compared to the target branch, not the other way + # around. The linex below with merge_base is + # equivalent to diff with three dots (git diff + # branch1...branch2) From the git documentation: + # "git diff A...B" is equivalent to "git diff + # $(git-merge-base A B) B" + repo.merge_base_commit(head, base) + end + + options ||= {} + actual_options = filter_diff_options(options) + repo.diff(common_commit, head, actual_options, *paths) + end + + # Return a copy of the +options+ hash containing only keys that can be + # passed to Rugged. Allowed options are: + # + # :max_size :: + # An integer specifying the maximum byte size of a file before a it + # will be treated as binary. The default value is 512MB. + # + # :context_lines :: + # The number of unchanged lines that define the boundary of a hunk + # (and to display before and after the actual changes). The default is + # 3. + # + # :interhunk_lines :: + # The maximum number of unchanged lines between hunk boundaries before + # the hunks will be merged into a one. The default is 0. + # + # :old_prefix :: + # The virtual "directory" to prefix to old filenames in hunk headers. + # The default is "a". + # + # :new_prefix :: + # The virtual "directory" to prefix to new filenames in hunk headers. + # The default is "b". + # + # :reverse :: + # If true, the sides of the diff will be reversed. + # + # :force_text :: + # If true, all files will be treated as text, disabling binary + # attributes & detection. + # + # :ignore_whitespace :: + # If true, all whitespace will be ignored. + # + # :ignore_whitespace_change :: + # If true, changes in amount of whitespace will be ignored. + # + # :ignore_whitespace_eol :: + # If true, whitespace at end of line will be ignored. + # + # :ignore_submodules :: + # if true, submodules will be excluded from the diff completely. + # + # :patience :: + # If true, the "patience diff" algorithm will be used (currenlty + # unimplemented). + # + # :include_ignored :: + # If true, ignored files will be included in the diff. + # + # :include_untracked :: + # If true, untracked files will be included in the diff. + # + # :include_unmodified :: + # If true, unmodified files will be included in the diff. + # + # :recurse_untracked_dirs :: + # Even if +:include_untracked+ is true, untracked directories will + # only be marked with a single entry in the diff. If this flag is set + # to true, all files under ignored directories will be included in the + # diff, too. + # + # :disable_pathspec_match :: + # If true, the given +*paths+ will be applied as exact matches, + # instead of as fnmatch patterns. + # + # :deltas_are_icase :: + # If true, filename comparisons will be made with case-insensitivity. + # + # :include_untracked_content :: + # if true, untracked content will be contained in the the diff patch + # text. + # + # :skip_binary_check :: + # If true, diff deltas will be generated without spending time on + # binary detection. This is useful to improve performance in cases + # where the actual file content difference is not needed. + # + # :include_typechange :: + # If true, type changes for files will not be interpreted as deletion + # of the "old file" and addition of the "new file", but will generate + # typechange records. + # + # :include_typechange_trees :: + # Even if +:include_typechange+ is true, blob -> tree changes will + # still usually be handled as a deletion of the blob. If this flag is + # set to true, blob -> tree changes will be marked as typechanges. + # + # :ignore_filemode :: + # If true, file mode changes will be ignored. + # + # :recurse_ignored_dirs :: + # Even if +:include_ignored+ is true, ignored directories will only be + # marked with a single entry in the diff. If this flag is set to true, + # all files under ignored directories will be included in the diff, + # too. + def filter_diff_options(options, default_options = {}) + allowed_options = [:max_size, :context_lines, :interhunk_lines, + :old_prefix, :new_prefix, :reverse, :force_text, + :ignore_whitespace, :ignore_whitespace_change, + :ignore_whitespace_eol, :ignore_submodules, + :patience, :include_ignored, :include_untracked, + :include_unmodified, :recurse_untracked_dirs, + :disable_pathspec_match, :deltas_are_icase, + :include_untracked_content, :skip_binary_check, + :include_typechange, :include_typechange_trees, + :ignore_filemode, :recurse_ignored_dirs, :paths, + :max_files, :max_lines, :all_diffs, :no_collapse] + + if default_options + actual_defaults = default_options.dup + actual_defaults.keep_if do |key| + allowed_options.include?(key) + end + else + actual_defaults = {} + end + + if options + filtered_opts = options.dup + filtered_opts.keep_if do |key| + allowed_options.include?(key) + end + filtered_opts = actual_defaults.merge(filtered_opts) + else + filtered_opts = actual_defaults + end + + filtered_opts + end + end + + def initialize(raw_diff, collapse: false) + case raw_diff + when Hash + init_from_hash(raw_diff, collapse: collapse) + when Rugged::Patch, Rugged::Diff::Delta + init_from_rugged(raw_diff, collapse: collapse) + when nil + raise "Nil as raw diff passed" + else + raise "Invalid raw diff type: #{raw_diff.class}" + end + end + + def serialize_keys + @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) + end + + def to_hash + hash = {} + + keys = serialize_keys + + keys.each do |key| + hash[key] = send(key) + end + + hash + end + + def submodule? + a_mode == '160000' || b_mode == '160000' + end + + def line_count + @line_count ||= Util.count_lines(@diff) + end + + def too_large? + if @too_large.nil? + @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + else + @too_large + end + end + + def collapsible? + @diff.bytesize >= DIFF_COLLAPSE_LIMIT + end + + def prune_large_diff! + @diff = '' + @line_count = 0 + @too_large = true + end + + def collapsed? + return @collapsed if defined?(@collapsed) + false + end + + def prune_collapsed_diff! + @diff = '' + @line_count = 0 + @collapsed = true + end + + private + + def init_from_rugged(rugged, collapse: false) + if rugged.is_a?(Rugged::Patch) + init_from_rugged_patch(rugged, collapse: collapse) + d = rugged.delta + else + d = rugged + end + + @new_path = encode!(d.new_file[:path]) + @old_path = encode!(d.old_file[:path]) + @a_mode = d.old_file[:mode].to_s(8) + @b_mode = d.new_file[:mode].to_s(8) + @new_file = d.added? + @renamed_file = d.renamed? + @deleted_file = d.deleted? + end + + def init_from_rugged_patch(patch, collapse: false) + # Don't bother initializing diffs that are too large. If a diff is + # binary we're not going to display anything so we skip the size check. + return if !patch.delta.binary? && prune_large_patch(patch, collapse) + + @diff = encode!(strip_diff_headers(patch.to_s)) + end + + def init_from_hash(hash, collapse: false) + raw_diff = hash.symbolize_keys + + serialize_keys.each do |key| + send(:"#{key}=", raw_diff[key.to_sym]) + end + + prune_large_diff! if too_large? + prune_collapsed_diff! if collapse && collapsible? + end + + # If the patch surpasses any of the diff limits it calls the appropiate + # prune method and returns true. Otherwise returns false. + def prune_large_patch(patch, collapse) + size = 0 + + patch.each_hunk do |hunk| + hunk.each_line do |line| + size += line.content.bytesize + + if size >= DIFF_SIZE_LIMIT + prune_large_diff! + return true + end + end + end + + if collapse && size >= DIFF_COLLAPSE_LIMIT + prune_collapsed_diff! + return true + end + + false + end + + # Strip out the information at the beginning of the patch's text to match + # Grit's output + def strip_diff_headers(diff_text) + # Delete everything up to the first line that starts with '---' or + # 'Binary' + diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') + + if diff_text.start_with?('---', 'Binary') + diff_text + else + # If the diff_text did not contain a line starting with '---' or + # 'Binary', return the empty string. No idea why; we are just + # preserving behavior from before the refactor. + '' + end + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb new file mode 100644 index 00000000000..65e06f5065d --- /dev/null +++ b/lib/gitlab/git/diff_collection.rb @@ -0,0 +1,129 @@ +module Gitlab + module Git + class DiffCollection + include Enumerable + + DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + + def initialize(iterator, options = {}) + @iterator = iterator + @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + @max_bytes = @max_files * 5120 # Average 5 KB per file + @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min + @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min + @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file + @all_diffs = !!options.fetch(:all_diffs, false) + @no_collapse = !!options.fetch(:no_collapse, true) + @deltas_only = !!options.fetch(:deltas_only, false) + + @line_count = 0 + @byte_count = 0 + @overflow = false + @array = Array.new + end + + def each(&block) + if @populated + # @iterator.each is slower than just iterating the array in place + @array.each(&block) + elsif @deltas_only + each_delta(&block) + else + each_patch(&block) + end + end + + def empty? + !@iterator.any? + end + + def overflow? + populate! + !!@overflow + end + + def size + @size ||= count # forces a loop using each method + end + + def real_size + populate! + + if @overflow + "#{size}+" + else + size.to_s + end + end + + def decorate! + collection = each_with_index do |element, i| + @array[i] = yield(element) + end + @populated = true + collection + end + + private + + def populate! + return if @populated + + each { nil } # force a loop through all diffs + @populated = true + nil + end + + def over_safe_limits?(files) + files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + end + + def each_delta + @iterator.each_delta.with_index do |delta, i| + diff = Gitlab::Git::Diff.new(delta) + + yield @array[i] = diff + end + end + + def each_patch + @iterator.each_with_index do |raw, i| + # First yield cached Diff instances from @array + if @array[i] + yield @array[i] + next + end + + # We have exhausted @array, time to create new Diff instances or stop. + break if @overflow + + if !@all_diffs && i >= @max_files + @overflow = true + break + end + + collapse = !@all_diffs && !@no_collapse + + diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + + if collapse && over_safe_limits?(i) + diff.prune_collapsed_diff! + end + + @line_count += diff.line_count + @byte_count += diff.diff.bytesize + + if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + break + end + + yield @array[i] = diff + end + end + end + end +end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb new file mode 100644 index 00000000000..e57d228e688 --- /dev/null +++ b/lib/gitlab/git/encoding_helper.rb @@ -0,0 +1,58 @@ +module Gitlab + module Git + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb new file mode 100644 index 00000000000..0148cd8df05 --- /dev/null +++ b/lib/gitlab/git/path_helper.rb @@ -0,0 +1,16 @@ +module Gitlab + module Git + class PathHelper + class << self + def normalize_path(filename) + # Strip all leading slashes so that //foo -> foo + filename[/^\/*/] = '' + + # Expand relative paths (e.g. foo/../bar) + filename = Pathname.new(filename) + filename.relative_path_from(Pathname.new('')) + end + end + end + end +end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb new file mode 100644 index 00000000000..df9ca3ee5ac --- /dev/null +++ b/lib/gitlab/git/popen.rb @@ -0,0 +1,26 @@ +require 'open3' + +module Gitlab + module Git + module Popen + def popen(cmd, path) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + vars = { "PWD" => path } + options = { chdir: path } + + @cmd_output = "" + @cmd_status = 0 + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_output << stdout.read + @cmd_output << stderr.read + @cmd_status = wait_thr.value.exitstatus + end + + [@cmd_output, @cmd_status] + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb new file mode 100644 index 00000000000..37ef6836742 --- /dev/null +++ b/lib/gitlab/git/ref.rb @@ -0,0 +1,49 @@ +module Gitlab + module Git + class Ref + include Gitlab::Git::EncodingHelper + + # Branch or tag name + # without "refs/tags|heads" prefix + attr_reader :name + + # Target sha. + # Usually it is commit sha but in case + # when tag reference on other tag it can be tag sha + attr_reader :target + + # Dereferenced target + # Commit object to which the Ref points to + attr_reader :dereferenced_target + + # Extract branch name from full ref path + # + # Ex. + # Ref.extract_branch_name('refs/heads/master') #=> 'master' + def self.extract_branch_name(str) + str.gsub(/\Arefs\/heads\//, '') + end + + def self.dereference_object(object) + object = object.target while object.is_a?(Rugged::Tag::Annotation) + + object + end + + def initialize(repository, name, target) + encode! name + @name = name.gsub(/\Arefs\/(tags|heads)\//, '') + @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + @target = if target.respond_to?(:oid) + target.oid + elsif target.respond_to?(:name) + target.name + elsif target.is_a? String + target + else + nil + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb new file mode 100644 index 00000000000..7068e68a855 --- /dev/null +++ b/lib/gitlab/git/repository.rb @@ -0,0 +1,1251 @@ +# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object +require 'forwardable' +require 'tempfile' +require 'forwardable' +require "rubygems/package" + +module Gitlab + module Git + class Repository + extend Forwardable + include Gitlab::Git::Popen + + SEARCH_CONTEXT_LINES = 3 + + class NoRepository < StandardError; end + class InvalidBlobName < StandardError; end + class InvalidRef < StandardError; end + + # Full path to repo + attr_reader :path + + # Directory name of repo + attr_reader :name + + # Rugged repo object + attr_reader :rugged + + # 'path' must be the path to a _bare_ git repository, e.g. + # /path/to/my-repo.git + def initialize(path) + @path = path + @name = path.split("/").last + @attributes = Gitlab::Git::Attributes.new(path) + end + + # Default branch in the repository + def root_ref + @root_ref ||= discover_default_branch + end + + # Alias to old method for compatibility + def raw + rugged + end + + def rugged + @rugged ||= Rugged::Repository.new(path) + rescue Rugged::RepositoryError, Rugged::OSError + raise NoRepository.new('no repository for such path') + end + + # Returns an Array of branch names + # sorted by name ASC + def branch_names + branches.map(&:name) + end + + # Returns an Array of Branches + def branches + rugged.branches.map do |rugged_ref| + begin + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact.sort_by(&:name) + end + + def reload_rugged + @rugged = nil + end + + # Directly find a branch with a simple name (e.g. master) + # + # force_reload causes a new Rugged repository to be instantiated + # + # This is to work around a bug in libgit2 that causes in-memory refs to + # be stale/invalid when packed-refs is changed. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + def find_branch(name, force_reload = false) + reload_rugged if force_reload + + rugged_ref = rugged.branches[name] + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + end + + def local_branches + rugged.branches.each(:local).map do |branch| + Gitlab::Git::Branch.new(self, branch.name, branch.target) + end + end + + # Returns the number of valid branches + def branch_count + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + + # Returns an Array of tag names + def tag_names + rugged.tags.map { |t| t.name } + end + + # Returns an Array of Tags + def tags + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + end.sort_by(&:name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + + # Returns an Array of branch and tag names + def ref_names + branch_names + tag_names + end + + # Deprecated. Will be removed in 5.2 + def heads + rugged.references.each("refs/heads/*").map do |head| + Gitlab::Git::Ref.new(self, head.name, head.target) + end.sort_by(&:name) + end + + def has_commits? + !empty? + end + + def empty? + rugged.empty? + end + + def bare? + rugged.bare? + end + + def repo_exists? + !!rugged + end + + # Discovers the default branch based on the repository's available branches + # + # - If no branches are present, returns nil + # - If one branch is present, returns its name + # - If two or more branches are present, returns current HEAD or master or first branch + def discover_default_branch + names = branch_names + + return if names.empty? + + return names[0] if names.length == 1 + + if rugged_head + extracted_name = Ref.extract_branch_name(rugged_head.name) + + return extracted_name if names.include?(extracted_name) + end + + if names.include?('master') + 'master' + else + names[0] + end + end + + def rugged_head + rugged.head + rescue Rugged::ReferenceError + nil + end + + def archive_metadata(ref, storage_path, format = "tar.gz") + ref ||= root_ref + commit = Gitlab::Git::Commit.find(self, ref) + return {} if commit.nil? + + project_name = self.name.chomp('.git') + prefix = "#{project_name}-#{ref}-#{commit.id}" + + { + 'RepoPath' => path, + 'ArchivePrefix' => prefix, + 'ArchivePath' => archive_file_path(prefix, storage_path, format), + 'CommitId' => commit.id, + } + end + + def archive_file_path(name, storage_path, format = "tar.gz") + # Build file path + return nil unless name + + extension = + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + "tar.bz2" + when "tar" + "tar" + when "zip" + "zip" + else + # everything else should fall back to tar.gz + "tar.gz" + end + + file_name = "#{name}.#{extension}" + File.join(storage_path, self.name, file_name) + end + + # Return repo size in megabytes + def size + size = popen(%w(du -sk), path).first.strip.to_i + (size.to_f / 1024).round(2) + end + + # Returns an array of BlobSnippets for files at the specified +ref+ that + # contain the +query+ string. + def search_files(query, ref = nil) + greps = [] + ref ||= root_ref + + populated_index(ref).each do |entry| + # Discard submodules + next if submodule?(entry) + + blob = Gitlab::Git::Blob.raw(self, entry[:oid]) + + # Skip binary files + next if blob.data.encoding == Encoding::ASCII_8BIT + + blob.load_all_data!(self) + greps += build_greps(blob.data, query, ref, entry[:path]) + end + + greps + end + + # Use the Rugged Walker API to build an array of commits. + # + # Usage. + # repo.log( + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # after: Time.new(2016, 4, 21, 14, 32, 10) + # ) + # + def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_DATE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + def log_by_shell(sha, options) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} log) + cmd += %W(-n #{options[:limit].to_i}) + cmd += %w(--format=%H) + cmd += %W(--skip=#{options[:offset].to_i}) + cmd += %w(--follow) if options[:follow] + cmd += %w(--no-merges) if options[:skip_merges] + cmd += %W(--after=#{options[:after].iso8601}) if options[:after] + cmd += %W(--before=#{options[:before].iso8601}) if options[:before] + cmd += [sha] + cmd += %W(-- #{options[:path]}) if options[:path].present? + + raw_output = IO.popen(cmd) {|io| io.read } + + log = raw_output.lines.map do |c| + Rugged::Commit.new(rugged, c.strip) + end + + log.is_a?(Array) ? log : [] + end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + # Return a collection of Rugged::Commits between the two revspec arguments. + # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for + # a detailed list of valid arguments. + def commits_between(from, to) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + sha_from = sha_from_ref(from) + sha_to = sha_from_ref(to) + + walker.push(sha_to) + walker.hide(sha_from) + + commits = walker.to_a + walker.reset + + commits + end + + # Counts the amount of commits between `from` and `to`. + def count_commits_between(from, to) + commits_between(from, to).size + end + + # Returns the SHA of the most recent common ancestor of +from+ and +to+ + def merge_base_commit(from, to) + rugged.merge_base(from, to) + end + + # Return an array of Diff objects that represent the diff + # between +from+ and +to+. See Diff::filter_diff_options for the allowed + # diff options. The +options+ hash can also include :break_rewrites to + # split larger rewrites into delete/add pairs. + def diff(from, to, options = {}, *paths) + Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + end + + # Returns commits collection + # + # Ex. + # repo.find_commits( + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :contains is the commit contained by the refs from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :date(default) or :topo + # + def find_commits(options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :contains, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + elsif actual_options[:contains] + branches_contains(actual_options[:contains]).each do |branch| + walker.push(branch.target_id) + end + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + if actual_options[:order] == :topo + walker.sorting(Rugged::SORT_TOPO) + else + walker.sorting(Rugged::SORT_DATE) + end + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + gitlab_commit = Gitlab::Git::Commit.decorate(commit) + commits.push(gitlab_commit) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + # Returns branch names collection that contains the special commit(SHA1 + # or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branch_names_contains(commit) + branches_contains(commit).map { |c| c.name } + end + + # Returns branch collection that contains the special commit(SHA1 or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branches_contains(commit) + commit_obj = rugged.rev_parse(commit) + parent = commit_obj.parents.first unless commit_obj.parents.empty? + + walker = Rugged::Walker.new(rugged) + + rugged.branches.select do |branch| + walker.push(branch.target_id) + walker.hide(parent) if parent + result = walker.any? { |c| c.oid == commit_obj.oid } + walker.reset + + result + end + end + + # Get refs hash which key is SHA1 + # and value is a Rugged::Reference + def refs_hash + # Initialize only when first call + if @refs_hash.nil? + @refs_hash = Hash.new { |h, k| h[k] = [] } + + rugged.references.each do |r| + # Symbolic/remote references may not have an OID; skip over them + target_oid = r.target.try(:oid) + if target_oid + sha = rev_parse_target(target_oid).oid + @refs_hash[sha] << r + end + end + end + @refs_hash + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + + # Return hash with submodules info for this repository + # + # Ex. + # { + # "rack" => { + # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", + # "path" => "rack", + # "url" => "git://github.com/chneukirchen/rack.git" + # }, + # "encoding" => { + # "id" => .... + # } + # } + # + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parse_gitmodules(commit, content) + end + + # Return total commits count accessible from passed ref + def commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + + # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or + # tag name or a commit SHA. Valid +reset_type+ values are: + # + # [:soft] + # the head will be moved to the commit. + # [:mixed] + # will trigger a +:soft+ reset, plus the index will be replaced + # with the content of the commit tree. + # [:hard] + # will trigger a +:mixed+ reset and the working directory will be + # replaced with the content of the index. (Untracked and ignored files + # will be left alone) + def reset(ref, reset_type) + rugged.reset(ref, reset_type) + end + + # Mimic the `git clean` command and recursively delete untracked files. + # Valid keys that can be passed in the +options+ hash are: + # + # :d - Remove untracked directories + # :f - Remove untracked directories that are managed by a different + # repository + # :x - Remove ignored files + # + # The value in +options+ must evaluate to true for an option to take + # effect. + # + # Examples: + # + # repo.clean(d: true, f: true) # Enable the -d and -f options + # + # repo.clean(d: false, x: true) # -x is enabled, -d is not + def clean(options = {}) + strategies = [:remove_untracked] + strategies.push(:force) if options[:f] + strategies.push(:remove_ignored) if options[:x] + + # TODO: implement this method + end + + # Check out the specified ref. Valid options are: + # + # :b - Create a new branch at +start_point+ and set HEAD to the new + # branch. + # + # * These options are passed to the Rugged::Repository#checkout method: + # + # :progress :: + # A callback that will be executed for checkout progress notifications. + # Up to 3 parameters are passed on each execution: + # + # - The path to the last updated file (or +nil+ on the very first + # invocation). + # - The number of completed checkout steps. + # - The number of total checkout steps to be performed. + # + # :notify :: + # A callback that will be executed for each checkout notification + # types specified with +:notify_flags+. Up to 5 parameters are passed + # on each execution: + # + # - An array containing the +:notify_flags+ that caused the callback + # execution. + # - The path of the current file. + # - A hash describing the baseline blob (or +nil+ if it does not + # exist). + # - A hash describing the target blob (or +nil+ if it does not exist). + # - A hash describing the workdir blob (or +nil+ if it does not + # exist). + # + # :strategy :: + # A single symbol or an array of symbols representing the strategies + # to use when performing the checkout. Possible values are: + # + # :none :: + # Perform a dry run (default). + # + # :safe :: + # Allow safe updates that cannot overwrite uncommitted data. + # + # :safe_create :: + # Allow safe updates plus creation of missing files. + # + # :force :: + # Allow all updates to force working directory to look like index. + # + # :allow_conflicts :: + # Allow checkout to make safe updates even if conflicts are found. + # + # :remove_untracked :: + # Remove untracked files not in index (that are not ignored). + # + # :remove_ignored :: + # Remove ignored files not in index. + # + # :update_only :: + # Only update existing files, don't create new ones. + # + # :dont_update_index :: + # Normally checkout updates index entries as it goes; this stops + # that. + # + # :no_refresh :: + # Don't refresh index/config/etc before doing checkout. + # + # :disable_pathspec_match :: + # Treat pathspec as simple list of exact match file paths. + # + # :skip_locked_directories :: + # Ignore directories in use, they will be left empty. + # + # :skip_unmerged :: + # Allow checkout to skip unmerged files (NOT IMPLEMENTED). + # + # :use_ours :: + # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). + # + # :use_theirs :: + # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). + # + # :update_submodules :: + # Recursively checkout submodules with same options (NOT + # IMPLEMENTED). + # + # :update_submodules_if_changed :: + # Recursively checkout submodules if HEAD moved in super repo (NOT + # IMPLEMENTED). + # + # :disable_filters :: + # If +true+, filters like CRLF line conversion will be disabled. + # + # :dir_mode :: + # Mode for newly created directories. Default: +0755+. + # + # :file_mode :: + # Mode for newly created files. Default: +0755+ or +0644+. + # + # :file_open_flags :: + # Mode for opening files. Default: + # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. + # + # :notify_flags :: + # A single symbol or an array of symbols representing the cases in + # which the +:notify+ callback should be invoked. Possible values are: + # + # :none :: + # Do not invoke the +:notify+ callback (default). + # + # :conflict :: + # Invoke the callback for conflicting paths. + # + # :dirty :: + # Invoke the callback for "dirty" files, i.e. those that do not need + # an update but no longer match the baseline. + # + # :updated :: + # Invoke the callback for any file that was changed. + # + # :untracked :: + # Invoke the callback for untracked files. + # + # :ignored :: + # Invoke the callback for ignored files. + # + # :all :: + # Invoke the callback for all these cases. + # + # :paths :: + # A glob string or an array of glob strings specifying which paths + # should be taken into account for the checkout operation. +nil+ will + # match all files. Default: +nil+. + # + # :baseline :: + # A Rugged::Tree that represents the current, expected contents of the + # workdir. Default: +HEAD+. + # + # :target_directory :: + # A path to an alternative workdir directory in which the checkout + # should be performed. + def checkout(ref, options = {}, start_point = "HEAD") + if options[:b] + rugged.branches.create(ref, start_point) + options.delete(:b) + end + default_options = { strategy: [:recreate_missing, :safe] } + rugged.checkout(ref, default_options.merge(options)) + end + + # Delete the specified branch from the repository + def delete_branch(branch_name) + rugged.branches.delete(branch_name) + end + + # Create a new branch named **ref+ based on **stat_point+, HEAD by default + # + # Examples: + # create_branch("feature") + # create_branch("other-feature", "master") + def create_branch(ref, start_point = "HEAD") + rugged_ref = rugged.branches.create(ref, start_point) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + + # Return an array of this repository's remote names + def remote_names + rugged.remotes.each_name.to_a + end + + # Delete the specified remote from this repository. + def remote_delete(remote_name) + rugged.remotes.delete(remote_name) + end + + # Add a new remote to this repository. Returns a Rugged::Remote object + def remote_add(remote_name, url) + rugged.remotes.create(remote_name, url) + end + + # Update the specified remote using the values in the +options+ hash + # + # Example + # repo.update_remote("origin", url: "path/to/repo") + def remote_update(remote_name, options = {}) + # TODO: Implement other remote options + rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + end + + # Fetch the specified remote + def fetch(remote_name) + rugged.remotes[remote_name].fetch + end + + # Push +*refspecs+ to the remote identified by +remote_name+. + def push(remote_name, *refspecs) + rugged.remotes[remote_name].push(refspecs) + end + + # Merge the +source_name+ branch into the +target_name+ branch. This is + # equivalent to `git merge --no_ff +source_name+`, since a merge commit + # is always created. + def merge(source_name, target_name, options = {}) + our_commit = rugged.branches[target_name].target + their_commit = rugged.branches[source_name].target + + raise "Invalid merge target" if our_commit.nil? + raise "Invalid merge source" if their_commit.nil? + + merge_index = rugged.merge_commits(our_commit, their_commit) + return false if merge_index.conflicts? + + actual_options = options.merge( + parents: [our_commit, their_commit], + tree: merge_index.write_tree(rugged), + update_ref: "refs/heads/#{target_name}" + ) + Rugged::Commit.create(rugged, actual_options) + end + + def commits_since(from_date) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + + commits = [] + walker.each do |commit| + break if commit.author[:time].to_date < from_date + commits.push(commit) + end + + commits + end + + AUTOCRLF_VALUES = { + "true" => true, + "false" => false, + "input" => :input + }.freeze + + def autocrlf + AUTOCRLF_VALUES[rugged.config['core.autocrlf']] + end + + def autocrlf=(value) + rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + end + + # Create a new directory with a .gitkeep file. Creates + # all required nested directories (i.e. mkdir -p behavior) + # + # options should contain next structure: + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + def mkdir(path, options = {}) + # Check if this directory exists; if it does, then don't bother + # adding .gitkeep file. + ref = options[:commit][:branch] + path = Gitlab::Git::PathHelper.normalize_path(path).to_s + rugged_ref = rugged.ref(ref) + + raise InvalidRef.new("Invalid ref") if rugged_ref.nil? + + target_commit = rugged_ref.target + + raise InvalidRef.new("Invalid target commit") if target_commit.nil? + + entry = tree_entry(target_commit, path) + + if entry + if entry[:type] == :blob + raise InvalidBlobName.new("Directory already exists as a file") + else + raise InvalidBlobName.new("Directory already exists") + end + end + + options[:file] = { + content: '', + path: "#{path}/.gitkeep", + update: true + } + + Gitlab::Git::Blob.commit(self, options) + end + + # Returns result like "git ls-files" , recursive and full file path + # + # Ex. + # repo.ls_files('master') + # + def ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end + + def copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end + + # Checks if the blob should be diffable according to its attributes + def diffable?(blob) + attributes(blob.path).fetch('diff') { blob.text? } + end + + # Returns the Git attributes for the given file path. + # + # See `Gitlab::Git::Attributes` for more information. + def attributes(path) + @attributes.attributes(path) + end + + private + + # Get the content of a blob for a given commit. If the blob is a commit + # (for submodules) then return the blob's OID. + def blob_content(commit, blob_name) + blob_entry = tree_entry(commit, blob_name) + + unless blob_entry + raise InvalidBlobName.new("Invalid blob name: #{blob_name}") + end + + case blob_entry[:type] + when :commit + blob_entry[:oid] + when :tree + raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") + when :blob + rugged.lookup(blob_entry[:oid]).content + end + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information. + def parse_gitmodules(commit, content) + results = {} + + current = "" + content.split("\n").each do |txt| + if txt =~ /^\s*\[/ + current = txt.match(/(?<=").*(?=")/)[0] + results[current] = {} + else + next unless results[current] + match_data = txt.match(/(\w+)\s*=\s*(.*)/) + next unless match_data + target = match_data[2].chomp + results[current][match_data[1]] = target + + if match_data[1] == "path" + begin + results[current]["id"] = blob_content(commit, target) + rescue InvalidBlobName + results.delete(current) + end + end + end + end + + results + end + + # Returns true if +commit+ introduced changes to +path+, using commit + # trees to make that determination. Uses the history simplification + # rules that `git log` uses by default, where a commit is omitted if it + # is TREESAME to any parent. + # + # If the +follow+ option is true and the file specified by +path+ was + # renamed, then the path value is set to the old path. + def commit_touches_path?(commit, path, follow, walker) + entry = tree_entry(commit, path) + + if commit.parents.empty? + # This is the root commit, return true if it has +path+ in its tree + return !entry.nil? + end + + num_treesame = 0 + commit.parents.each do |parent| + parent_entry = tree_entry(parent, path) + + # Only follow the first TREESAME parent for merge commits + if num_treesame > 0 + walker.hide(parent) + next + end + + if entry.nil? && parent_entry.nil? + num_treesame += 1 + elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] + num_treesame += 1 + end + end + + case num_treesame + when 0 + detect_rename(commit, commit.parents.first, path) if follow + true + else false + end + end + + # Find the entry for +path+ in the tree for +commit+ + def tree_entry(commit, path) + pathname = Pathname.new(path) + first = true + tmp_entry = nil + + pathname.each_filename do |dir| + if first + tmp_entry = commit.tree[dir] + first = false + elsif tmp_entry.nil? + return nil + else + tmp_entry = rugged.lookup(tmp_entry[:oid]) + return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] + end + end + + tmp_entry + end + + # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was + # renamed in +commit+, then set +path+ to the old filename. + def detect_rename(commit, parent, path) + diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) + + # If +path+ is a filename, not a directory, then we should only have + # one delta. We don't need to follow renames for directories. + return nil if diff.each_delta.count > 1 + + delta = diff.each_delta.first + if delta.added? + full_diff = parent.diff(commit) + full_diff.find_similar! + + full_diff.each_delta do |full_delta| + if full_delta.renamed? && path == full_delta.new_file[:path] + # Look for the old path in ancestors + path.replace(full_delta.old_file[:path]) + end + end + end + end + + def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) + git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) + + # Put files into a directory before archiving + prefix = "#{archive_name(treeish)}/" + git_archive_cmd << "--prefix=#{prefix}" + + # Format defaults to tar + git_archive_cmd << "--format=#{format}" if format + + git_archive_cmd += %W(-- #{treeish}) + + open(filename, 'w') do |file| + # Create a pipe to act as the '|' in 'git archive ... | gzip' + pipe_rd, pipe_wr = IO.pipe + + # Get the compression process ready to accept data from the read end + # of the pipe + compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) + # The read end belongs to the compression process now; we should + # close our file descriptor for it. + pipe_rd.close + + # Start 'git archive' and tell it to write into the write end of the + # pipe. + git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) + # The write end belongs to 'git archive' now; close it. + pipe_wr.close + + # When 'git archive' and the compression process are finished, we are + # done. + Process.waitpid(git_archive_pid) + raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? + Process.waitpid(compress_pid) + raise "#{compress_cmd.join(' ')} failed" unless $?.success? + end + end + + def nice(cmd) + nice_cmd = %w(nice -n 20) + unless unsupported_platform? + nice_cmd += %w(ionice -c 2 -n 7) + end + nice_cmd + cmd + end + + def unsupported_platform? + %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? + end + + # Returns true if the index entry has the special file mode that denotes + # a submodule. + def submodule?(index_entry) + index_entry[:mode] == 57344 + end + + # Return a Rugged::Index that has read from the tree at +ref_name+ + def populated_index(ref_name) + commit = rev_parse_target(ref_name) + index = rugged.index + index.read_tree(commit.tree) + index + end + + # Return an array of BlobSnippets for lines in +file_contents+ that match + # +query+ + def build_greps(file_contents, query, ref, filename) + # The file_contents string is potentially huge so we make sure to loop + # through it one line at a time. This gives Ruby the chance to GC lines + # we are not interested in. + # + # We need to do a little extra work because we are not looking for just + # the lines that matches the query, but also for the context + # (surrounding lines). We will use Enumerable#each_cons to efficiently + # loop through the lines while keeping surrounding lines on hand. + # + # First, we turn "foo\nbar\nbaz" into + # [ + # [nil, -3], [nil, -2], [nil, -1], + # ['foo', 0], ['bar', 1], ['baz', 3], + # [nil, 4], [nil, 5], [nil, 6] + # ] + lines_with_index = Enumerator.new do |yielder| + # Yield fake 'before' lines for the first line of file_contents + (-SEARCH_CONTEXT_LINES..-1).each do |i| + yielder.yield [nil, i] + end + + # Yield the actual file contents + count = 0 + file_contents.each_line do |line| + line.chomp! + yielder.yield [line, count] + count += 1 + end + + # Yield fake 'after' lines for the last line of file_contents + (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| + yielder.yield [nil, i] + end + end + + greps = [] + + # Loop through consecutive blocks of lines with indexes + lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| + # Get the 'middle' line and index from the block + line, _ = line_block[SEARCH_CONTEXT_LINES] + + next unless line && line.match(/#{Regexp.escape(query)}/i) + + # Yay, 'line' contains a match! + # Get an array with just the context lines (no indexes) + match_with_context = line_block.map(&:first) + # Remove 'nil' lines in case we are close to the first or last line + match_with_context.compact! + + # Get the line number (1-indexed) of the first context line + first_context_line_number = line_block[0][1] + 1 + + greps << Gitlab::Git::BlobSnippet.new( + ref, + match_with_context, + first_context_line_number, + filename + ) + end + + greps + end + + # Return the Rugged patches for the diff between +from+ and +to+. + def diff_patches(from, to, options = {}, *paths) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) + + diff = rugged.diff(from, to, actual_options) + diff.find_similar!(break_rewrites: break_rewrites) + diff.each_patch + end + end + end +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb new file mode 100644 index 00000000000..b5342c3d310 --- /dev/null +++ b/lib/gitlab/git/tag.rb @@ -0,0 +1,17 @@ +module Gitlab + module Git + class Tag < Ref + attr_reader :object_sha + + def initialize(repository, name, target, message = nil) + super(repository, name, target) + + @message = message + end + + def message + encode! @message + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb new file mode 100644 index 00000000000..f7450e8b58f --- /dev/null +++ b/lib/gitlab/git/tree.rb @@ -0,0 +1,104 @@ +module Gitlab + module Git + class Tree + include Gitlab::Git::EncodingHelper + + attr_accessor :id, :root_id, :name, :path, :type, + :mode, :commit_id, :submodule_url + + class << self + # Get list of tree objects + # for repository based on commit sha and path + # Uses rugged for raw objects + def where(repository, sha, path = nil) + path = nil if path == '' || path == '/' + + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode], + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha, + ) + end + end + + # Recursive search of tree id for path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # views/ # oid: 4a + # + # + # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a' + # + def find_id_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] && entry[:type] == :tree + end + + return nil unless entry + + if path_arr.size > 1 + path_arr.shift + find_id_by_path(repository, entry[:oid], path_arr.join('/')) + else + entry[:oid] + end + end + end + + def initialize(options) + %w(id root_id name path type mode commit_id).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + end + + def name + encode! @name + end + + def dir? + type == :tree + end + + def file? + type == :blob + end + + def submodule? + type == :commit + end + + def readme? + name =~ /^readme/i + end + + def contributing? + name =~ /^contributing/i + end + end + end +end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb new file mode 100644 index 00000000000..7973da2e8f8 --- /dev/null +++ b/lib/gitlab/git/util.rb @@ -0,0 +1,18 @@ +module Gitlab + module Git + module Util + LINE_SEP = "\n".freeze + + def self.count_lines(string) + case string[-1] + when nil + 0 + when LINE_SEP + string.count(LINE_SEP) + else + string.count(LINE_SEP) + 1 + end + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 3f635be22ba..a55adc9b1c8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class ProjectCreator + include Gitlab::CurrentSettings + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -34,7 +36,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end # diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 4d4e04e9e35..b8a5ac907a4 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,6 +13,7 @@ module Gitlab if current_user gon.current_user_id = current_user.id + gon.current_username = current_user.username end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index e6ecd118609..08ad3274b38 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -6,6 +6,7 @@ project_tree: - :events - issues: - :events + - :timelogs - notes: - :author - :events @@ -27,6 +28,7 @@ project_tree: - :events - :merge_request_diff - :events + - :timelogs - label_links: - label: :priorities diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b790733f4a7..2405b94db50 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,13 +1,10 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids - def initialize(exported_members:, user:, project:) - @exported_members = exported_members + @exported_members = user.admin? ? exported_members : [] @user = user @project = project - @missing_author_ids = [] # This needs to run first, as second call would be from #map # which means project members already exist. @@ -39,7 +36,6 @@ module Gitlab def missing_keys_tracking_hash Hash.new do |_, key| - @missing_author_ids << key default_user_id end end @@ -64,7 +60,7 @@ module Gitlab end def find_project_user_query(member) - user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 7a649f28340..19e43cce768 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -80,17 +80,13 @@ module Gitlab # is left. def set_note_author old_author_id = @relation_hash['author_id'] - - # Users with admin access can map users - @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) end - def missing_author?(old_author_id) - !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + def has_author?(old_author_id) + admin_user? && @members_mapper.map.keys.include?(old_author_id) end def missing_author_note(updated_at, author_name) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 45958710c13..52276cbcd9a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -5,8 +5,6 @@ # module Gitlab module ImportSources - extend CurrentSettings - ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 801dfde9a36..b91012d6405 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,5 +1,6 @@ module Gitlab module IncomingEmail + UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self @@ -18,7 +19,11 @@ module Gitlab end def reply_address(key) - config.address.gsub(WILDCARD_PLACEHOLDER, key) + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + def unsubscribe_address(key) + config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end def key_from_address(address) @@ -49,7 +54,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') Regexp.new(regex).freeze end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb new file mode 100644 index 00000000000..8db91d25a4b --- /dev/null +++ b/lib/gitlab/job_waiter.rb @@ -0,0 +1,27 @@ +module Gitlab + # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + class JobWaiter + # The sleep interval between checking keys, in seconds. + INTERVAL = 0.1 + + # jobs - The job IDs to wait for. + def initialize(jobs) + @jobs = jobs + end + + # Waits for all the jobs to be completed. + # + # timeout - The maximum amount of seconds to block the caller for. This + # ensures we don't indefinitely block a caller in case a job takes + # long to process, or is never processed. + def wait(timeout = 60) + start = Time.current + + while (Time.current - start) <= timeout + break if SidekiqStatus.all_completed?(@jobs) + + sleep(INTERVAL) # to not overload Redis too much. + end + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 7e06bd2b0fb..54a5b1d31cd 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -34,21 +34,21 @@ module Gitlab def allowed? if ldap_user unless ldap_config.active_directory - user.activate if user.ldap_blocked? + unblock_user(user, 'is available again') if user.ldap_blocked? return true end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.ldap_block + block_user(user, 'is disabled in Active Directory') false else - user.activate if user.ldap_blocked? + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.ldap_block + block_user(user, 'does not exist anymore') false end end @@ -64,6 +64,24 @@ module Gitlab def ldap_user @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end end end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index bf4dd9542d5..95378e5a769 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -25,7 +25,7 @@ module Gitlab end def get_raw(key) - auth_hash.extra[:raw_info][key] + auth_hash.extra[:raw_info][key] if auth_hash.extra end def ldap_config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index de52ef3fc65..28129198438 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -107,7 +107,7 @@ module Gitlab end def attributes - options['attributes'] + default_attributes.merge(options['attributes']) end def timeout @@ -130,6 +130,16 @@ module Gitlab end end + def default_attributes + { + 'username' => %w(uid userid sAMAccountName), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + protected def base_options diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 333f170a484..7084fd1767d 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -28,7 +28,7 @@ module Gitlab end def name - attribute_value(:name) + attribute_value(:name).first end def uid @@ -62,14 +62,12 @@ module Gitlab # this method looks for 'mail', 'email' and 'userPrincipalName' and # returns the first with a value. def attribute_value(attribute) - attributes = Array(config.attributes[attribute.to_sym]) + attributes = Array(config.attributes[attribute.to_s]) selected_attr = attributes.find { |attr| entry.respond_to?(attr) } return nil unless selected_attr - # Some LDAP attributes return an array, - # even if it is a single value (like 'cn') - Array(entry.public_send(selected_attr)).first + entry.public_send(selected_attr) end end end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 91fb0bb317a..47f88727fc8 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -70,8 +70,19 @@ module Gitlab def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.request_method][endpoint.route.path] - trans.action = "Grape##{endpoint.route.request_method} #{path}" + + begin + route = endpoint.route + rescue + # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + end + + if route + path = endpoint_paths_cache[route.request_method][route.path] + trans.action = "Grape##{route.request_method} #{path}" + end end private diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6bdf3db9cb8..db325c00705 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -71,6 +71,14 @@ module Gitlab ) end + def single_commit_result? + commits_count == 1 && total_result_count == 1 + end + + def total_result_count + issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + end + private def blobs @@ -114,7 +122,25 @@ module Gitlab end def commits - @commits ||= project.repository.find_commits_by_message(query) + @commits ||= find_commits(query) + end + + def find_commits(query) + return [] unless Ability.allowed?(@current_user, :download_code, @project) + + commits = find_commits_by_message(query) + commit_by_sha = find_commit_by_sha(query) + commits |= [commit_by_sha] if commit_by_sha + commits + end + + def find_commits_by_message(query) + project.repository.find_commits_by_message(query) + end + + def find_commit_by_sha(query) + key = query.strip + project.repository.commit(key) if Commit.valid_hash?(key) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9226da2d6b1..9384102acec 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -42,7 +42,7 @@ module Gitlab return @_raw_config if defined?(@_raw_config) begin - @_raw_config = File.read(CONFIG_FILE).freeze + @_raw_config = ERB.new(File.read(CONFIG_FILE)).result.freeze rescue Errno::ENOENT @_raw_config = false end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 9e0b0e5ea98..a3fa7c1331a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,11 +61,11 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'." + "can contain only letters, digits, '_', '-', '@', '+' and '.'." end def file_path_regex diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 35212992698..c9c65f76f4b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,10 @@ module Gitlab @milestones_count ||= milestones.count end + def single_commit_result? + false + end + private def projects diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb new file mode 100644 index 00000000000..aadc401ff8d --- /dev/null +++ b/lib/gitlab/sidekiq_status.rb @@ -0,0 +1,66 @@ +module Gitlab + # The SidekiqStatus module and its child classes can be used for checking if a + # Sidekiq job has been processed or not. + # + # To check if a job has been completed, simply pass the job ID to the + # `completed?` method: + # + # job_id = SomeWorker.perform_async(...) + # + # if Gitlab::SidekiqStatus.completed?(job_id) + # ... + # end + # + # For each job ID registered a separate key is stored in Redis, making lookups + # much faster than using Sidekiq's built-in job finding/status API. These keys + # expire after a certain period of time to prevent storing too many keys in + # Redis. + module SidekiqStatus + STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + + # The default time (in seconds) after which a status key is expired + # automatically. The default of 30 minutes should be more than sufficient + # for most jobs. + DEFAULT_EXPIRATION = 30.minutes.to_i + + # Starts tracking of the given job. + # + # jid - The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.set(jid, expire = DEFAULT_EXPIRATION) + Sidekiq.redis do |redis| + redis.set(key_for(jid), 1, ex: expire) + end + end + + # Stops the tracking of the given job. + # + # jid - The Sidekiq job ID to remove. + def self.unset(jid) + Sidekiq.redis do |redis| + redis.del(key_for(jid)) + end + end + + # Returns true if all the given job have been completed. + # + # jids - The Sidekiq job IDs to check. + # + # Returns true or false. + def self.all_completed?(jids) + keys = jids.map { |jid| key_for(jid) } + + responses = Sidekiq.redis do |redis| + redis.pipelined do + keys.each { |key| redis.exists(key) } + end + end + + responses.all? { |value| !value } + end + + def self.key_for(jid) + STATUS_KEY % jid + end + end +end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb new file mode 100644 index 00000000000..779a9998b22 --- /dev/null +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -0,0 +1,10 @@ +module Gitlab + module SidekiqStatus + class ClientMiddleware + def call(_, job, _, _) + SidekiqStatus.set(job['jid']) + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb new file mode 100644 index 00000000000..31dfa46ff9d --- /dev/null +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqStatus + class ServerMiddleware + def call(worker, job, queue) + ret = yield + + SidekiqStatus.unset(job['jid']) + + ret + end + end + end +end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb new file mode 100644 index 00000000000..d615c24149a --- /dev/null +++ b/lib/gitlab/time_tracking_formatter.rb @@ -0,0 +1,34 @@ +module Gitlab + module TimeTrackingFormatter + extend self + + def parse(string) + with_custom_config do + string.sub!(/\A-/, '') + + seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil + seconds *= -1 if seconds && Regexp.last_match + seconds + end + end + + def output(seconds) + with_custom_config do + ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil + end + end + + def with_custom_config + # We may want to configure it through project settings in a future version. + ChronicDuration.hours_per_day = 8 + ChronicDuration.days_per_week = 5 + + result = yield + + ChronicDuration.hours_per_day = 24 + ChronicDuration.days_per_week = 7 + + result + end + end +end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 6c7e673fb9f..6ce9b229294 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -35,7 +35,9 @@ module Gitlab return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + has_access = access_levels.any? { |access_level| access_level.check_access(user) } + + has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) else user.can?(:push_code, project) end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb new file mode 100644 index 00000000000..83c8ba5c1cf --- /dev/null +++ b/lib/gitlab/view/presenter/base.rb @@ -0,0 +1,28 @@ +module Gitlab + module View + module Presenter + module Base + extend ActiveSupport::Concern + + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject + + def can?(user, action, overriden_subject = nil) + super(user, action, overriden_subject || subject) + end + + class_methods do + def presenter? + true + end + + def presents(name) + define_method(name) { subject } + end + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb new file mode 100644 index 00000000000..f4d330c590e --- /dev/null +++ b/lib/gitlab/view/presenter/delegated.rb @@ -0,0 +1,19 @@ +module Gitlab + module View + module Presenter + class Delegated < SimpleDelegator + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + + super(subject) + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb new file mode 100644 index 00000000000..d172d61e2c9 --- /dev/null +++ b/lib/gitlab/view/presenter/factory.rb @@ -0,0 +1,24 @@ +module Gitlab + module View + module Presenter + class Factory + def initialize(subject, **attributes) + @subject = subject + @attributes = attributes + end + + def fabricate! + presenter_class.new(subject, attributes) + end + + private + + attr_reader :subject, :attributes + + def presenter_class + "#{subject.class.name}Presenter".constantize + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb new file mode 100644 index 00000000000..b7653a0f3cc --- /dev/null +++ b/lib/gitlab/view/presenter/simple.rb @@ -0,0 +1,17 @@ +module Gitlab + module View + module Presenter + class Simple + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + end + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..c7953af29dd 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,6 +11,7 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index d28bb583fe7..a3b502ffd6a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -15,10 +15,17 @@ module Gitlab class << self def git_http_ok(repository, user) - { + params = { GL_ID: Gitlab::GlId.gl_id(user), RepoPath: repository.path_to_repo, } + + params.merge!( + GitalySocketPath: Gitlab.config.gitaly.socket_path, + GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", + ) if Gitlab.config.gitaly.socket_path.present? + + params end def lfs_upload_ok(oid, size) diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index ec2903b7ec6..e55c0d6ac49 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -8,21 +8,31 @@ module Mattermost @user = user end - private - def with_session(&blk) Mattermost::Session.new(user).with_session(&blk) end - def json_get(path, options = {}) + private + + # Should be used in a session manually + def get(session, path, options = {}) + json_response session.get(path, options) + end + + # Should be used in a session manually + def post(session, path, options = {}) + json_response session.post(path, options) + end + + def session_get(path, options = {}) with_session do |session| - json_response session.get(path, options) + get(session, path, options) end end - def json_post(path, options = {}) + def session_post(path, options = {}) with_session do |session| - json_response session.post(path, options) + post(session, path, options) end end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb index d1e4bb0eccf..33e450d7f0a 100644 --- a/lib/mattermost/command.rb +++ b/lib/mattermost/command.rb @@ -1,7 +1,7 @@ module Mattermost class Command < Client def create(params) - response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create", body: params.to_json) response['token'] diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 784eca6ab5a..09dfd082b3a 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,7 +1,7 @@ module Mattermost class Team < Client def all - json_get('/api/v3/teams/all') + session_get('/api/v3/teams/all') end end end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 6f27972c4e4..5e94fba97bf 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -7,9 +7,4 @@ namespace :dev do Rake::Task["gitlab:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke end - - desc 'GitLab | Start/restart foreman and watch for changes' - task :foreman => :environment do - sh 'rerun --dir app,config,lib -- foreman start' - end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index f9834a4dae8..a67c1fe1f27 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -3,7 +3,7 @@ namespace :gitlab do desc "GitLab | Git | Repack" task repack: :environment do - failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".color(:green) else @@ -13,17 +13,17 @@ namespace :gitlab do desc "GitLab | Git | Run garbage collection on all repos" task gc: :environment do - failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".color(:green) else output_failures(failures) end end - + desc "GitLab | Git | Prune all repos" task prune: :environment do - failures = perform_git_cmd(%W(git prune), "Git Prune") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") if failures.empty? puts "Done".color(:green) else diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index dffea8ed155..f7c831892ee 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -11,8 +11,10 @@ namespace :gitlab do gem_version = run_command(%W(gem --version)) # check Bundler version bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) - # check Bundler version + # check Rake version rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) + # check redis version + redis_version = run_and_match(%W(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a puts "" puts "System information".color(:yellow) @@ -24,6 +26,7 @@ namespace :gitlab do puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" + puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 4f76dad7286..b77a5bb62d1 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -44,7 +44,7 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ ) ] |