diff options
Diffstat (limited to 'lib')
78 files changed, 1203 insertions, 441 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 4602e627fdb..74ca4728695 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -31,6 +31,7 @@ module API mount ::API::AccessRequests mount ::API::AwardEmoji mount ::API::Branches + mount ::API::BroadcastMessages mount ::API::Builds mount ::API::CommitStatuses mount ::API::Commits @@ -44,11 +45,13 @@ module API mount ::API::Keys mount ::API::Labels mount ::API::LicenseTemplates + mount ::API::Lint mount ::API::Members mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes + mount ::API::NotificationSettings mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::ProjectSnippets diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2efe7e3adf3..7c22b17e4e5 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -54,7 +54,7 @@ module API post endpoint do required_attributes! [:name] - not_found!('Award Emoji') unless can_read_awardable? + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -92,6 +92,10 @@ module API can?(current_user, ability, awardable) end + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) + end + def awardable @awardable ||= begin diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb new file mode 100644 index 00000000000..fb2a4148011 --- /dev/null +++ b/lib/api/broadcast_messages.rb @@ -0,0 +1,99 @@ +module API + class BroadcastMessages < Grape::API + before { authenticate! } + before { authenticated_as_admin! } + + resource :broadcast_messages do + helpers do + def find_message + BroadcastMessage.find(params[:id]) + end + end + + desc 'Get all broadcast messages' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + optional :page, type: Integer, desc: 'Current page number' + optional :per_page, type: Integer, desc: 'Number of messages per page' + end + get do + messages = BroadcastMessage.all + + present paginate(messages), with: Entities::BroadcastMessage + end + + desc 'Create a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time', default: -> { Time.zone.now } + optional :ends_at, type: DateTime, desc: 'Ending time', default: -> { 1.hour.from_now } + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + post do + create_params = declared(params, include_missing: false).to_h + message = BroadcastMessage.create(create_params) + + if message.persisted? + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Get a specific broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + get ':id' do + message = find_message + + present message, with: Entities::BroadcastMessage + end + + desc 'Update a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + optional :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time' + optional :ends_at, type: DateTime, desc: 'Ending time' + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + put ':id' do + message = find_message + update_params = declared(params, include_missing: false).to_h + + if message.update(update_params) + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Delete a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + delete ':id' do + message = find_message + + present message.destroy, with: Entities::BroadcastMessage + end + end + end +end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 4df6ca8333e..dfbdd597d29 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -37,7 +37,7 @@ module API # id (required) - The ID of a project # sha (required) - The commit hash # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, error or failure + # state (required) - The state of the status. Can be: pending, running, success, failed or canceled # target_url (optional) - The target URL to associate with this status # description (optional) - A short description of the status # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" @@ -46,7 +46,7 @@ module API post ':id/statuses/:sha' do authorize! :create_commit_status, user_project required_attributes! [:state] - attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + attrs = attributes_for_keys [:target_url, :description] commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -58,36 +58,38 @@ module API # the first found branch on that commit ref = params[:ref] - unless ref - branches = @project.repository.branch_names_contains(commit.sha) - not_found! 'References for commit' if branches.none? - ref = branches.first - end + ref ||= @project.repository.branch_names_contains(commit.sha).first + not_found! 'References for commit' unless ref - pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) + name = params[:name] || params[:context] || 'default' - name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) - status.update(attrs) + pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) - case params[:state].to_s - when 'running' - status.run - when 'success' - status.success - when 'failed' - status.drop - when 'canceled' - status.cancel - else - status.status = params[:state].to_s - end + status = GenericCommitStatus.running_or_pending.find_or_initialize_by( + project: @project, pipeline: pipeline, + user: current_user, name: name, ref: ref) + status.attributes = attrs + + begin + case params[:state].to_s + when 'pending' + status.enqueue! + when 'running' + status.enqueue + status.run! + when 'success' + status.success! + when 'failed' + status.drop! + when 'canceled' + status.cancel! + else + render_api_error!('invalid state', 400) + end - if status.save present status, with: Entities::CommitStatus - else - render_validation_error!(status) + rescue StateMachines::InvalidTransition => e + render_api_error!(e.message, 400) end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cbb324dd06d..bfee4b6c752 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -76,15 +76,24 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :container_registry_enabled + + # Expose old field names with the new permissions methods to keep API compatible + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) } + expose :created_at, :last_activity_at expose :shared_runners_enabled + expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| @@ -113,6 +122,7 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level + expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url end @@ -211,6 +221,7 @@ module API expose :user_notes_count expose :upvotes, :downvotes expose :due_date + expose :confidential expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) @@ -232,6 +243,8 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :diff_head_sha, as: :sha + expose :merge_commit_sha expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end @@ -364,7 +377,7 @@ module API expose :access_level expose :notification_level do |member, options| if member.notification_setting - NotificationSetting.levels[member.notification_setting.level] + ::NotificationSetting.levels[member.notification_setting.level] end end end @@ -375,6 +388,21 @@ module API class GroupAccess < MemberAccess end + class NotificationSetting < Grape::Entity + expose :level + expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do + ::NotificationSetting::EMAIL_EVENTS.each do |event| + expose event + end + end + end + + class GlobalNotificationSetting < NotificationSetting + expose :notification_email do |notification_setting, options| + notification_setting.user.notification_email + end + end + class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events @@ -574,5 +602,10 @@ module API class Template < Grape::Entity expose :name, :content end + + class BroadcastMessage < Grape::Entity + expose :id, :message, :starts_at, :ends_at, :color, :font + expose :active?, as: :active + end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9d8b8d737a9..60ac9bdfa33 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -27,13 +27,14 @@ module API # path (required) - The path of the group # description (optional) - The description of the group # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group # Example Request: # POST /groups post do - authorize! :create_group, current_user + authorize! :create_group required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled] @group = Group.new(attrs) if @group.save @@ -51,13 +52,14 @@ module API # path (optional) - The path of the group # description (optional) - The description of the group # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group # Example Request: # PUT /groups/:id put ':id' do group = find_group(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled] if ::Groups::UpdateService.new(group, current_user, attrs).execute present group, with: Entities::GroupDetail @@ -97,7 +99,7 @@ module API group = find_group(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects - present projects, with: Entities::Project + present projects, with: Entities::Project, user: current_user end # Transfer a project to the Group namespace diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index da4b1bf9902..150875ed4f0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -129,7 +129,7 @@ module API forbidden! unless current_user.is_admin? end - def authorize!(action, subject) + def authorize!(action, subject = nil) forbidden! unless can?(current_user, action, subject) end @@ -148,7 +148,7 @@ module API end def can?(object, action, subject) - abilities.allowed?(object, action, subject) + Ability.allowed?(object, action, subject) end # Checks the occurrences of required attributes, each attribute must be present in the params hash @@ -269,6 +269,10 @@ module API render_api_error!('304 Not Modified', 304) end + def no_content! + render_api_error!('204 No Content', 204) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -408,14 +412,6 @@ module API links.join(', ') end - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end - def secret_token File.read(Gitlab.config.gitlab_shell.secret_file).chomp end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 5b54c11ef62..1114fd21784 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -35,6 +35,14 @@ module API Project.find_with_namespace(project_path) end end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end end post "/allowed" do @@ -51,9 +59,9 @@ module API access = if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol) + Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol) + Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) end access_status = access.check(params[:action], params[:changes]) @@ -105,15 +113,19 @@ module API post '/two_factor_recovery_codes' do status 200 - key = Key.find(params[:key_id]) - user = key.user + key = Key.find_by(id: params[:key_id]) - # Make sure this isn't a deploy key - unless key.type.nil? + unless key + return { 'success' => false, 'message' => 'Could not find the given key' } + end + + if key.is_a?(DeployKey) return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } end - unless user.present? + user = key.user + + unless user return { success: false, message: 'Could not find a user for the given key' } end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 077258faee1..c9689e6f8ef 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -41,7 +41,8 @@ module API issues = current_user.issues.inc_notes_with_associations issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -73,7 +74,11 @@ module API params[:group_id] = group.id params[:milestone_title] = params.delete(:milestone) params[:label_name] = params.delete(:labels) - params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + if params[:order_by] || params[:sort] + # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example) + params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}" + end issues = IssuesFinder.new(current_user, params).execute @@ -113,7 +118,8 @@ module API issues = filter_issues_milestone(issues, params[:milestone]) end - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end @@ -140,12 +146,13 @@ module API # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # POST /projects/:id/issues post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -154,21 +161,19 @@ module API render_api_error!({ labels: errors }, 400) end - project = user_project + attrs[:labels] = params[:labels] if params[:labels] + + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? - issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute + issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels].present? - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) @@ -188,12 +193,13 @@ module API # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # PUT /projects/:id/issues/:issue_id put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -202,17 +208,15 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? + issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels] && can?(current_user, :admin_issue, user_project) - issue.remove_labels - # Create and add labels to the new created issue - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) diff --git a/lib/api/lint.rb b/lib/api/lint.rb new file mode 100644 index 00000000000..ae43a4a3237 --- /dev/null +++ b/lib/api/lint.rb @@ -0,0 +1,21 @@ +module API + class Lint < Grape::API + namespace :ci do + desc 'Validation of .gitlab-ci.yml content' + params do + requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + end + post '/lint' do + error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + + status 200 + + if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + end + end + end +end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8bfa998dc53..c5c214d4d13 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -83,12 +83,12 @@ module API opts[:created_at] = params[:created_at] end - @note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::const_get(note.class.name) else - not_found!("Note #{@note.errors.messages}") + not_found!("Note #{note.errors.messages}") end end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb new file mode 100644 index 00000000000..a70a7e71073 --- /dev/null +++ b/lib/api/notification_settings.rb @@ -0,0 +1,97 @@ +module API + # notification_settings API + class NotificationSettings < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + resource :notification_settings do + desc 'Get global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + get do + notification_setting = current_user.global_notification_setting + + present notification_setting, with: Entities::GlobalNotificationSetting + end + + desc 'Update global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + params do + optional :level, type: String, desc: 'The global notification level' + optional :notification_email, type: String, desc: 'The email address to send notifications' + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put do + notification_setting = current_user.global_notification_setting + + begin + notification_setting.transaction do + new_notification_email = params.delete(:notification_email) + declared_params = declared(params, include_missing: false).to_h + + current_user.update(notification_email: new_notification_email) if new_notification_email + notification_setting.update(declared_params) + end + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! current_user + render_validation_error! notification_setting + present notification_setting, with: Entities::GlobalNotificationSetting + end + end + + %w[group project].each do |source_type| + resource source_type.pluralize do + desc "Get #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + end + get ":id/notification_settings" do + source = find_source(source_type, params[:id]) + + notification_setting = current_user.notification_settings_for(source) + + present notification_setting, with: Entities::NotificationSetting + end + + desc "Update #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + optional :level, type: String, desc: "The #{source_type} notification level" + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put ":id/notification_settings" do + source = find_source(source_type, params.delete(:id)) + notification_setting = current_user.notification_settings_for(source) + + begin + declared_params = declared(params, include_missing: false).to_h + + notification_setting.update(declared_params) + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! notification_setting + present notification_setting, with: Entities::NotificationSetting + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 2aae75c471d..2a0c8e1f2c0 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -13,11 +13,14 @@ module API params do optional :page, type: Integer, desc: 'Page number of the current request' optional :per_page, type: Integer, desc: 'Number of items per page' + optional :scope, type: String, values: ['running', 'branches', 'tags'], + desc: 'Either running, branches, or tags' end get ':id/pipelines' do authorize! :read_pipeline, user_project - present paginate(user_project.pipelines), with: Entities::Pipeline + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: Entities::Pipeline end desc 'Gets a specific pipeline for the project' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 71efd4f33ca..644d836ed0b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -51,7 +51,7 @@ module API @projects = current_user.viewable_starred_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::Project + present @projects, with: Entities::Project, user: current_user end # Get all projects for admin user @@ -105,6 +105,7 @@ module API # visibility_level (optional) - 0 by default # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) # Example Request # POST /projects post do @@ -124,7 +125,8 @@ module API :visibility_level, :import_url, :public_builds, - :only_allow_merge_if_build_succeeds] + :only_allow_merge_if_build_succeeds, + :lfs_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -156,6 +158,7 @@ module API # visibility_level (optional) # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -174,7 +177,8 @@ module API :visibility_level, :import_url, :public_builds, - :only_allow_merge_if_build_succeeds] + :only_allow_merge_if_build_succeeds, + :lfs_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -185,16 +189,30 @@ module API end end - # Fork new project for the current user. + # Fork new project for the current user or provided namespace. # # Parameters: # id (required) - The ID of a project + # namespace (optional) - The ID or name of the namespace that the project will be forked into. # Example Request # POST /projects/fork/:id post 'fork/:id' do + attrs = {} + namespace_id = params[:namespace] + + if namespace_id.present? + namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) + + not_found!('Target Namespace') unless namespace + + attrs[:namespace] = namespace + end + @forked_project = ::Projects::ForkService.new(user_project, - current_user).execute + current_user, + attrs).execute + if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else @@ -220,6 +238,7 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project # public_builds (optional) + # lfs_enabled (optional) # Example Request # PUT /projects/:id put ':id' do @@ -237,7 +256,8 @@ module API :public, :visibility_level, :public_builds, - :only_allow_merge_if_build_succeeds] + :only_allow_merge_if_build_succeeds, + :lfs_enabled] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -408,18 +428,9 @@ module API # Example Request: # GET /projects/search/:query get "/search/:query" do - ids = current_user.authorized_projects.map(&:id) - visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] - projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") - sort = params[:sort] == 'desc' ? 'desc' : 'asc' - - projects = case params["order_by"] - when 'id' then projects.order("id #{sort}") - when 'name' then projects.order("name #{sort}") - when 'created_at' then projects.order("created_at #{sort}") - when 'last_activity_at' then projects.order("last_activity_at #{sort}") - else projects - end + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(project_order_by => project_sort) present paginate(projects), with: Entities::Project end diff --git a/lib/api/users.rb b/lib/api/users.rb index 8a376d3c2a3..c440305ff0f 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -327,7 +327,7 @@ module API # Example Request: # GET /user get do - present @current_user, with: Entities::UserLogin + present @current_user, with: Entities::UserFull end # Get currently authenticated user's keys diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index f117fc3d37d..9fcd9a3f999 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -55,7 +55,7 @@ module Backup bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) FileUtils.mv(path, bk_repos_path) # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 2770) + FileUtils.mkdir_p(path, mode: 02770) end Project.find_each(batch_size: 1000) do |project| diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index d77a5e3ff09..16cd774c81a 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.object_class_title - @object_title ||= object_class.name.titleize - end - # Public: Find references in text (like `!123` for merge requests) # # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| @@ -49,10 +45,6 @@ module Banzai self.class.object_sym end - def object_class_title - self.class.object_class_title - end - def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -198,7 +190,7 @@ module Banzai end def object_link_title(object) - "#{object_class_title}: #{object.title}" + object.title end def object_link_text(object, matches) diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index bbb88c979cc..4358bf45549 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -35,7 +35,7 @@ module Banzai end def object_link_title(range) - range.reference_title + nil end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 2ce1816672b..a26dd09c25a 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -28,10 +28,6 @@ module Banzai only_path: context[:only_path]) end - def object_link_title(commit) - commit.link_title - end - def object_link_text_extras(object, matches) extras = super diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e258dc8e2bf..8f262ef3d8d 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -70,6 +70,11 @@ module Banzai def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end + + def object_link_title(object) + # use title of wrapped element instead + nil + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index ca686c87d97..58fff496d00 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -59,6 +59,10 @@ module Banzai html_safe end end + + def object_link_title(object) + nil + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index bf058241cda..2d221290f7e 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -52,7 +52,7 @@ module Banzai end def reference_class(type) - "gfm gfm-#{type}" + "gfm gfm-#{type} has-tooltip" end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 2e2c8da311e..e7a1ec8457d 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -31,6 +31,7 @@ module Banzai def apply_relative_link_rules! if @uri.relative? && @uri.path.present? link = ::File.join(@wiki_base_path, @uri.path) + link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 6cf218aaa0d..e8e03e4a98f 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -211,7 +211,7 @@ module Banzai end def can?(user, permission, subject) - Ability.abilities.allowed?(user, permission, subject) + Ability.allowed?(user, permission, subject) end def find_projects_for_hash_keys(hash) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 9f3b582a263..59f85416ee5 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,7 +12,7 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact + update_runner_last_contact(save: false) update_runner_info required_attributes! [:token] not_found! unless current_runner.active? @@ -27,7 +27,7 @@ module Ci else Gitlab::Metrics.add_event(:build_not_found) - not_found! + build_not_found! end end @@ -101,6 +101,7 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build @@ -113,7 +114,8 @@ module Ci end status 200 - { TempPath: ArtifactUploader.artifacts_upload_path } + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok end # Upload artifacts to build - Runners only diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..66c05773b68 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -15,6 +15,15 @@ module Ci expose :filename, :size end + class BuildOptions < Grape::Entity + expose :image + expose :services + expose :artifacts + expose :cache + expose :dependencies + expose :after_script + end + class Build < Grape::Entity expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 199d62d9b8a..23353c62885 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 60 + UPDATE_RUNNER_EVERY = 40 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -14,19 +14,37 @@ module Ci end def authenticate_build_token!(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - forbidden! unless token && build.valid_token?(token) + forbidden! unless build_token_valid?(build) end def runner_registration_token_valid? - params[:token] == current_application_settings.runners_registration_token + ActiveSupport::SecurityUtils.variable_size_secure_compare( + params[:token], + current_application_settings.runners_registration_token) + end + + def build_token_valid?(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + + # We require to also check `runners_token` to maintain compatibility with old version of runners + token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact + def update_runner_last_contact(save: true) # Use a random threshold to prevent beating DB updates + # it generates a distribution between: [40m, 80m] contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.update_attributes(contacted_at: Time.now) + current_runner.contacted_at = Time.now + current_runner.save if current_runner.changed? && save + end + end + + def build_not_found! + if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + no_content! + else + not_found! end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 47efd5bd9f2..0369e80312a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -55,29 +55,36 @@ module Ci { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - ## - # Refactoring note: - # - before script behaves differently than after script - # - after script returns an array of commands - # - before script should be a concatenated command - commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), + commands: job[:commands], tag_list: job[:tags] || [], name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], + environment: job[:environment_name], yaml_variables: yaml_variables(name), options: { - image: job[:image] || @image, - services: job[:services] || @services, + image: job[:image], + services: job[:services], artifacts: job[:artifacts], - cache: job[:cache] || @cache, + cache: job[:cache], dependencies: job[:dependencies], - after_script: job[:after_script] || @after_script, + after_script: job[:after_script], + environment: job[:environment], }.compact } end + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Ci::GitlabCiYamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + private def initial_parsing diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb new file mode 100644 index 00000000000..3da04edde70 --- /dev/null +++ b/lib/ci/mask_secret.rb @@ -0,0 +1,9 @@ +module Ci::MaskSecret + class << self + def mask(value, token) + return value unless value.present? && token.present? + + value.gsub(token, 'x' * token.length) + end + end +end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb new file mode 100644 index 00000000000..7b1533d0d32 --- /dev/null +++ b/lib/expand_variables.rb @@ -0,0 +1,17 @@ +module ExpandVariables + class << self + def expand(value, variables) + # Convert hash array to variables + if variables.is_a?(Array) + variables = variables.reduce({}) do |hash, variable| + hash[variable[:key]] = variable[:value] + hash + end + end + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables[$1 || $2] + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 91f0270818a..0a0f1c3b17b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,21 +1,21 @@ module Gitlab module Auth - Result = Struct.new(:user, :type) + class MissingPersonalTokenError < StandardError; end class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? - result = Result.new + result = + service_request_check(login, password, project) || + build_access_token_check(login, password) || + user_with_password_for_git(login, password) || + oauth_access_token_check(login, password) || + personal_access_token_check(login, password) || + Gitlab::Auth::Result.new - if valid_ci_request?(login, password, project) - result.type = :ci - else - result = populate_result(login, password) - end + rate_limit!(ip, success: result.success?, login: login) - success = result.user.present? || [:ci, :missing_personal_token].include?(result.type) - rate_limit!(ip, success: success, login: login) result end @@ -57,44 +57,31 @@ module Gitlab private - def valid_ci_request?(login, password, project) + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) - return false unless project && matched_login.present? + return unless project && matched_login.present? underscored_service = matched_login['service'].underscore - if underscored_service == 'gitlab_ci' - project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) + if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. service = project.public_send("#{underscored_service}_service") - service && service.activated? && service.valid_token?(password) - end - end - - def populate_result(login, password) - result = - user_with_password_for_git(login, password) || - oauth_access_token_check(login, password) || - personal_access_token_check(login, password) - - if result - result.type = nil unless result.user - - if result.user && result.user.two_factor_enabled? && result.type == :gitlab_or_ldap - result.type = :missing_personal_token + if service && service.activated? && service.valid_token?(password) + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) end end - - result || Result.new end def user_with_password_for_git(login, password) user = find_with_user_password(login, password) - Result.new(user, :gitlab_or_ldap) if user + return unless user + + raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + + Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end def oauth_access_token_check(login, password) @@ -102,7 +89,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if token && token.accessible? user = User.find_by(id: token.resource_owner_id) - Result.new(user, :oauth) + Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end end end @@ -111,9 +98,52 @@ module Gitlab if login && password user = User.find_by_personal_access_token(password) validation = User.by_login(login) - Result.new(user, :personal_token) if user == validation + Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + end + end + + def build_access_token_check(login, password) + return unless login == 'gitlab-ci-token' + return unless password + + build = ::Ci::Build.running.find_by_token(password) + return unless build + return unless build.project.builds_enabled? + + if build.user + # If user is assigned to build, use restricted credentials of user + Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) + else + # Otherwise use generic CI credentials (backward compatibility) + Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities) end end + + public + + def build_authentication_abilities + [ + :read_project, + :build_download_code, + :build_read_container_image, + :build_create_container_image + ] + end + + def read_authentication_abilities + [ + :read_project, + :download_code, + :read_container_image + ] + end + + def full_authentication_abilities + read_authentication_abilities + [ + :push_code, + :create_container_image + ] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb new file mode 100644 index 00000000000..bf625649cbf --- /dev/null +++ b/lib/gitlab/auth/result.rb @@ -0,0 +1,13 @@ +module Gitlab + module Auth + Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + def ci? + type == :ci + end + + def success? + actor.present? || type == :ci + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 839a4fa30d5..c412249a01e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -195,7 +195,7 @@ module Gitlab # Create (if necessary) and link the secret token file def generate_and_link_secret_token secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.exist? secret_file + unless File.size?(secret_file) # Generate a new token of 16 random hexadecimal characters and store it in secret_file. token = SecureRandom.hex(16) File.write(secret_file, token) diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 95d925dc7f3..9a0482306b7 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -12,9 +12,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.pipelines - .latest_successful_for(@ref) - .first + @pipeline = @project.pipelines.latest_successful_for(@ref) end def entity diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7beaecd1cf0..f4b5097adb1 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -21,7 +21,7 @@ module Gitlab private - def gl_user_id(project, bitbucket_id) + def gitlab_user_id(project, bitbucket_id) if bitbucket_id user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) (user && user.id) || project.creator_id @@ -74,7 +74,7 @@ module Gitlab description: body, title: issue["title"], state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, reporter) + author_id: gitlab_user_id(project, reporter) ) end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 4b32eb966aa..cb1065223d4 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -23,6 +23,7 @@ module Gitlab protected def protected_branch_checks + return unless @branch_name return unless project.protected_branch?(@branch_name) if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ae82c0db3f1..bbfa6cf7d05 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -14,7 +14,7 @@ module Gitlab @config = Loader.new(config).load! @global = Node::Global.new(@config) - @global.process! + @global.compose! end def valid? diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index 2de82d40c9d..6b7ab2fdaf2 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -23,9 +23,9 @@ module Gitlab end end - private + def compose!(deps = nil) + return unless valid? - def compose! self.class.nodes.each do |key, factory| factory .value(@config[key]) @@ -33,6 +33,12 @@ module Gitlab @entries[key] = factory.create! end + + yield if block_given? + + @entries.each_value do |entry| + entry.compose!(deps) + end end class_methods do diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 0c782c422b5..8717eabf81e 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -20,11 +20,14 @@ module Gitlab @validator.validate(:new) end - def process! + def [](key) + @entries[key] || Node::Undefined.new + end + + def compose!(deps = nil) return unless valid? - compose! - descendants.each(&:process!) + yield if block_given? end def leaf? @@ -73,11 +76,6 @@ module Gitlab def self.validator Validator end - - private - - def compose! - end end end end diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb new file mode 100644 index 00000000000..d388ab6b879 --- /dev/null +++ b/lib/gitlab/ci/config/node/environment.rb @@ -0,0 +1,68 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an environment. + # + class Environment < Entry + include Validatable + + ALLOWED_KEYS = %i[name url] + + validations do + validate do + unless hash? || string? + errors.add(:config, 'should be a hash or a string') + end + end + + validates :name, presence: true + validates :name, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message } + + validates :name, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + with_options if: :hash? do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :url, + length: { maximum: 255 }, + addressable_url: true, + allow_nil: true + end + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def url + value[:url] + end + + def value + case @config + when String then { name: @config } + when Hash then @config + else {} + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 707b052e6a8..5387f29ad59 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -37,8 +37,8 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Undefined.new( - fabricate_undefined + Node::Unspecified.new( + fabricate_unspecified ) else fabricate(@node, @value) @@ -47,13 +47,13 @@ module Gitlab private - def fabricate_undefined + def fabricate_unspecified ## # If node has a default value we fabricate concrete node # with default value. # if @node.default.nil? - fabricate(Node::Null) + fabricate(Node::Undefined) else fabricate(@node, @node.default) end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index ccd539fb003..2a2943c9288 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -36,15 +36,15 @@ module Gitlab helpers :before_script, :image, :services, :after_script, :variables, :stages, :types, :cache, :jobs - private - - def compose! - super - - compose_jobs! - compose_deprecated_entries! + def compose!(_deps = nil) + super(self) do + compose_jobs! + compose_deprecated_entries! + end end + private + def compose_jobs! factory = Node::Factory.new(Node::Jobs) .value(@config.except(*self.class.nodes.keys)) diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/node/hidden.rb index 073044b66f8..fe4ee8a7fc6 100644 --- a/lib/gitlab/ci/config/node/hidden_job.rb +++ b/lib/gitlab/ci/config/node/hidden.rb @@ -5,11 +5,10 @@ module Gitlab ## # Entry that represents a hidden CI/CD job. # - class HiddenJob < Entry + class Hidden < Entry include Validatable validations do - validates :config, type: Hash validates :config, presence: true end diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb index e84737acbb9..603334d6793 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/node/job.rb @@ -13,7 +13,7 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :environment, :dependencies + attributes :tags, :allow_failure, :when, :dependencies validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -29,58 +29,65 @@ module Gitlab inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ 'always or manual' } - validates :environment, - type: { - with: String, - message: Gitlab::Regex.environment_name_regex_message } - validates :environment, - format: { - with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } validates :dependencies, array_of_strings: true end end - node :before_script, Script, + node :before_script, Node::Script, description: 'Global before script overridden in this job.' - node :script, Commands, + node :script, Node::Commands, description: 'Commands that will be executed in this job.' - node :stage, Stage, + node :stage, Node::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Stage, + node :type, Node::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Script, + node :after_script, Node::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Cache, + node :cache, Node::Cache, description: 'Cache definition for this job.' - node :image, Image, + node :image, Node::Image, description: 'Image that will be used to execute this job.' - node :services, Services, + node :services, Node::Services, description: 'Services that will be used to execute this job.' - node :only, Trigger, + node :only, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Trigger, + node :except, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Variables, + node :variables, Node::Variables, description: 'Environment variables available for this job.' - node :artifacts, Artifacts, + node :artifacts, Node::Artifacts, description: 'Artifacts configuration for this job.' + node :environment, Node::Environment, + description: 'Environment configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts + :artifacts, :commands, :environment + + def compose!(deps = nil) + super do + if type_defined? && !stage_defined? + @entries[:stage] = @entries[:type] + end + + @entries.delete(:type) + end + + inherit!(deps) + end def name @metadata[:name] @@ -90,12 +97,30 @@ module Gitlab @config.merge(to_hash.compact) end + def commands + (before_script_value.to_a + script_value.to_a).join("\n") + end + private + def inherit!(deps) + return unless deps + + self.class.nodes.each_key do |key| + global_entry = deps[key] + job_entry = @entries[key] + + if global_entry.specified? && !job_entry.specified? + @entries[key] = global_entry + end + end + end + def to_hash { name: name, before_script: before_script, script: script, + commands: commands, image: image, services: services, stage: stage, @@ -103,19 +128,11 @@ module Gitlab only: only, except: except, variables: variables_defined? ? variables : nil, + environment: environment_defined? ? environment : nil, + environment_name: environment_defined? ? environment[:name] : nil, artifacts: artifacts, after_script: after_script } end - - def compose! - super - - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - end - - @entries.delete(:type) - end end end end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb index 51683c82ceb..d10e80d1a7d 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/node/jobs.rb @@ -26,19 +26,23 @@ module Gitlab name.to_s.start_with?('.') end - private - - def compose! - @config.each do |name, config| - node = hidden?(name) ? Node::HiddenJob : Node::Job - - factory = Node::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") + def compose!(deps = nil) + super do + @config.each do |name, config| + node = hidden?(name) ? Node::Hidden : Node::Job + + factory = Node::Factory.new(node) + .value(config || {}) + .metadata(name: name) + .with(key: name, parent: self, + description: "#{name} job definition.") + + @entries[name] = factory.create! + end - @entries[name] = factory.create! + @entries.each_value do |entry| + entry.compose!(deps) + end end end end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 88a5f53f13c..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents an undefined node. - # - # Implements the Null Object pattern. - # - class Null < Entry - def value - nil - end - - def valid? - true - end - - def errors - [] - end - - def specified? - false - end - - def relevant? - false - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb index 45fef8c3ae5..33e78023539 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -3,15 +3,34 @@ module Gitlab class Config module Node ## - # This class represents an unspecified entry node. + # This class represents an undefined node. # - # It decorates original entry adding method that indicates it is - # unspecified. + # Implements the Null Object pattern. # - class Undefined < SimpleDelegator + class Undefined < Entry + def initialize(*) + super(nil) + end + + def value + nil + end + + def valid? + true + end + + def errors + [] + end + def specified? false end + + def relevant? + false + end end end end diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/node/unspecified.rb new file mode 100644 index 00000000000..a7d1f6131b8 --- /dev/null +++ b/lib/gitlab/ci/config/node/unspecified.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an unspecified entry node. + # + # It decorates original entry adding method that indicates it is + # unspecified. + # + class Unspecified < SimpleDelegator + def specified? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb new file mode 100644 index 00000000000..a210e76acaa --- /dev/null +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -0,0 +1,141 @@ +module Gitlab + module Ci + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module PipelineDuration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest. + where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 2d4d55daeeb..98e842cded3 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -18,7 +18,7 @@ module Gitlab def parse(text, our_path:, their_path:, parent_file: nil) raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 102400 + raise UnmergeableFile if text.length > 200.kilobytes begin text.to_json diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index bd681f03173..b164f5a2eea 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,16 +1,16 @@ module Gitlab class ContributionsCalendar - attr_reader :timestamps, :projects, :user + attr_reader :activity_dates, :projects, :user def initialize(projects, user) @projects = projects @user = user end - def timestamps - return @timestamps if @timestamps.present? + def activity_dates + return @activity_dates if @activity_dates.present? - @timestamps = {} + @activity_dates = {} date_from = 1.year.ago events = Event.reorder(nil).contributions.where(author_id: user.id). @@ -19,18 +19,17 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a + activity_dates = (1.year.ago.to_date..Date.today).to_a - dates.each do |date| - date_id = date.to_time.to_i.to_s + activity_dates.each do |date| day_events = events.find { |day_events| day_events["date"] == date } if day_events - @timestamps[date_id] = day_events["total_amount"] + @activity_dates[date] = day_events["total_amount"] end end - @timestamps + @activity_dates end def events_by_date(date) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 927f9dad20b..0bd6e148ba8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -129,12 +129,14 @@ module Gitlab # column - The name of the column to add. # type - The column type (e.g. `:integer`). # default - The default value for the column. + # limit - Sets a column limit. For example, for :integer, the default is + # 4-bytes. Set `limit: 8` to allow 8-byte integers. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. # # This method can also take a block which is passed directly to the # `update_column_in_batches` method. - def add_column_with_default(table, column, type, default:, allow_null: false, &block) + def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -144,7 +146,11 @@ module Gitlab disable_statement_timeout transaction do - add_column(table, column, type, default: nil) + if limit + add_column(table, column, type, default: nil, limit: limit) + else + add_column(table, column, type, default: nil) + end # Changing the default before the update ensures any newly inserted # rows already use the proper default value. diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 9b681e636c7..bd90d24a2ec 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -17,11 +17,13 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return [true, nil] unless exists? - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + Bundler.with_clean_env do + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1882eb8d050..799794c0171 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -5,12 +5,13 @@ module Gitlab DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project, :protocol, :user_access + attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol + @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) end @@ -60,14 +61,26 @@ module Gitlab end def user_download_access_check - unless user_access.can_do_action?(:download_code) + unless user_can_download_code? || build_can_download_code? return build_status_object(false, "You are not allowed to download code from this project.") end build_status_object(true) end + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + def user_push_access_check(changes) + unless authentication_abilities.include?(:push_code) + return build_status_object(false, "You are not allowed to upload code for this project.") + end + if changes.blank? return build_status_object(true) end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 72992baffd4..8cacf4f4925 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,11 +15,16 @@ module Gitlab private - def gl_user_id(github_id) + def gitlab_user_id(github_id) User.joins(:identities). find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). try(:id) end + + def gitlab_author_id + return @gitlab_author_id if defined?(@gitlab_author_id) + @gitlab_author_id = gitlab_user_id(raw_data.user.id) + end end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 2c1b94ef2cd..2bddcde2b7c 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -21,7 +21,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -52,7 +52,11 @@ module Gitlab end def note - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def type diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 02ffb43d89b..d35ee2a1c65 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -24,6 +24,7 @@ module Gitlab import_issues import_pull_requests import_wiki + import_releases handle_errors true @@ -133,8 +134,7 @@ module Gitlab if issue.labels.count > 0 label_ids = issue.labels - .map { |raw| LabelFormatter.new(project, raw).attributes } - .map { |attrs| Label.find_by(attrs).try(:id) } + .map { |attrs| project.labels.find_by(title: attrs.name).try(:id) } .compact issuable.update_attribute(:label_ids, label_ids) @@ -152,12 +152,14 @@ module Gitlab end def create_comments(issuable, comments) - comments.each do |raw| - begin - comment = CommentFormatter.new(project, raw) - issuable.notes.create!(comment.attributes) - rescue => e - errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + ActiveRecord::Base.no_touching do + comments.each do |raw| + begin + comment = CommentFormatter.new(project, raw) + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end end end end @@ -166,7 +168,7 @@ module Gitlab unless project.wiki_enabled? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.update_attribute(:wiki_enabled, true) + project.project.update_attribute(:wiki_access_level, ProjectFeature::ENABLED) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, @@ -176,6 +178,18 @@ module Gitlab errors << { type: :wiki, errors: e.message } end end + + def import_releases + releases = client.releases(repo, per_page: 100) + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 835ec858b35..77621de9f4c 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -12,7 +12,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -40,7 +40,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -49,7 +49,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -57,7 +57,11 @@ module Gitlab end def description - @formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -69,10 +73,6 @@ module Gitlab def state raw_data.state == 'closed' ? 'closed' : 'opened' end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 9f18244e7d7..2cad7fca88e 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -13,6 +13,12 @@ module Gitlab Label end + def create! + project.labels.find_or_create_by!(title: title) do |label| + label.color = color + end + end + private def color diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 53d4b3102d1..b2fa524cf5b 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,14 +3,14 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: number, + iid: raw_data.number, project: project, - title: title, - description: description, - due_date: due_date, + title: raw_data.title, + description: raw_data.description, + due_date: raw_data.due_on, state: state, - created_at: created_at, - updated_at: updated_at + created_at: raw_data.created_at, + updated_at: raw_data.updated_at } end @@ -20,33 +20,9 @@ module Gitlab private - def number - raw_data.number - end - - def title - raw_data.title - end - - def description - raw_data.description - end - - def due_date - raw_data.due_on - end - def state raw_data.state == 'closed' ? 'closed' : 'active' end - - def created_at - raw_data.created_at - end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index f4221003db5..e9725880c5e 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -11,18 +11,24 @@ module Gitlab end def execute - ::Projects::CreateService.new( + project = ::Projects::CreateService.new( current_user, name: repo.name, path: repo.name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), - wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") ).execute + + # If repo has wiki we'll import it later + if repo.has_wiki? && project + project.project_feature.update_attribute(:wiki_access_level, ProjectFeature::DISABLED) + end + + project end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 04aa3664f64..1408683100f 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -20,7 +20,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -68,7 +68,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -77,7 +77,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -85,7 +85,11 @@ module Gitlab end def description - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -103,15 +107,6 @@ module Gitlab 'opened' end end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at - else - raw_data.updated_at - end - end end end end diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb new file mode 100644 index 00000000000..73d643b00ad --- /dev/null +++ b/lib/gitlab/github_import/release_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class ReleaseFormatter < BaseFormatter + def attributes + { + project: project, + tag: raw_data.tag_name, + description: raw_data.body, + created_at: raw_data.created_at, + updated_at: raw_data.created_at + } + end + + def klass + Release + end + + def valid? + !raw_data.draft + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 46d40f75be6..e44d7934fda 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -41,7 +41,8 @@ module Gitlab title: issue["title"], state: issue["state"], updated_at: issue["updated_at"], - author_id: gl_user_id(project, issue["author"]["id"]) + author_id: gitlab_user_id(project, issue["author"]["id"]), + confidential: issue["confidential"] ) end end @@ -51,7 +52,7 @@ module Gitlab private - def gl_user_id(project, gitlab_id) + def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb562bdcd2c..181e288a014 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.3' + # For every version update, the version history in import_export.md has to be kept up to date. + VERSION = '0.1.4' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1da51043611..925a952156f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -35,19 +35,18 @@ project_tree: - :deploy_keys - :services - :hooks - - :protected_branches + - protected_branches: + - :merge_access_levels + - :push_access_levels - :labels - milestones: - :events + - :project_feature # Only include the following attributes for the models specified. included_attributes: project: - :description - - :issues_enabled - - :merge_requests_enabled - - :wiki_enabled - - :snippets_enabled - :visibility_level - :archived user: @@ -72,4 +71,4 @@ methods: statuses: - :type merge_request_diff: - - :utf8_st_diffs
\ No newline at end of file + - :utf8_st_diffs diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b0726268ca6..354ccd64696 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -7,7 +7,9 @@ module Gitlab variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze @@ -17,6 +19,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + FINDER_ATTRIBUTES = %w[title project_id].freeze + def self.create(*args) new(*args).create end @@ -149,7 +153,7 @@ module Gitlab end def parsed_relation_hash - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } end def set_st_diffs @@ -161,14 +165,30 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) - existing_object.assign_attributes(parsed_relation_hash) + events = parsed_relation_hash.delete('events') + + unless events.blank? + existing_object.assign_attributes(events: events) + end + existing_object else relation_class.new(parsed_relation_hash) end end end + + def existing_object + @existing_object ||= + begin + finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES) + existing_object = relation_class.find_or_create_by(finder_hash) + # Done in two steps, as MySQL behaves differently than PostgreSQL using + # the +find_or_create_by+ method and does not return the ID the second time. + existing_object.update(parsed_relation_hash) + existing_object + end + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 6d9379acf25..d1e33ea8678 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -22,10 +22,6 @@ module Gitlab private - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def path_to_repo @project.repository.path_to_repo end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index de3fe6d822e..fc08082fc86 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,8 +24,8 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 9a5bcfb5c9b..9100719da87 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -23,31 +23,7 @@ module Gitlab end def users(field, value, limit = nil) - if field.to_sym == :dn - options = { - base: value, - scope: Net::LDAP::SearchScope_BaseObject - } - else - options = { - base: config.base, - filter: Net::LDAP::Filter.eq(field, value) - } - end - - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - - options[:filter] = if options[:filter] - Net::LDAP::Filter.join(options[:filter], user_filter) - else - user_filter - end - end - - if limit.present? - options.merge!(size: limit) - end + options = user_options(field, value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -90,6 +66,38 @@ module Gitlab Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") [] end + + private + + def user_options(field, value, limit) + options = { attributes: %W(#{config.uid} cn mail dn) } + options[:size] = limit if limit + + if field.to_sym == :dn + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + options[:filter] = user_filter + else + options[:base] = config.base + options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + end + + options + end + + def user_filter(filter = nil) + if config.user_filter.present? + user_filter = Net::LDAP::Filter.construct(config.user_filter) + end + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end end end end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index ca23ccef25b..cc74bb29087 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -18,18 +18,18 @@ module Gitlab FileUtils.mkdir_p(path) end - @cmd_output = "" - @cmd_status = 0 + cmd_output = "" + cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - # We are not using stdin so we should close it, in case the command we - # are running waits for input. + yield(stdin) if block_given? stdin.close - @cmd_output << stdout.read - @cmd_output << stderr.read - @cmd_status = wait_thr.value.exitstatus + + cmd_output << stdout.read + cmd_output << stderr.read + cmd_status = wait_thr.value.exitstatus end - [@cmd_output, @cmd_status] + [cmd_output, cmd_status] end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 183bd10d6a3..5b9cfaeb2f8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -28,11 +28,6 @@ module Gitlab end end - def total_count - @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + commits_count - end - def blobs_count @blobs_count ||= blobs.count end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ffad5e17c78..bc8bbf337f3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,11 +96,11 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index f8ab2b1f09e..2690938fe82 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -27,11 +27,6 @@ module Gitlab end end - def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + - milestones_count - end - def projects_count @projects_count ||= projects.count end @@ -48,10 +43,6 @@ module Gitlab @milestones_count ||= milestones.count end - def empty? - total_count.zero? - end - private def projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb new file mode 100644 index 00000000000..117fc508135 --- /dev/null +++ b/lib/gitlab/sentry.rb @@ -0,0 +1,27 @@ +module Gitlab + module Sentry + def self.enabled? + Rails.env.production? && current_application_settings.sentry_enabled? + end + + def self.context(current_user = nil) + return unless self.enabled? + + if current_user + Raven.user_context( + id: current_user.id, + email: current_user.email, + username: current_user.username, + ) + end + end + + def self.program_context + if Sidekiq.server? + 'sidekiq' + else + 'rails' + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e0e74ff8359..9e01f02029c 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -20,10 +20,6 @@ module Gitlab end end - def total_count - @total_count ||= snippet_titles_count + snippet_blobs_count - end - def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c6826a09bd2..60aae541d46 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,19 +1,38 @@ require 'base64' require 'json' +require 'securerandom' module Gitlab class Workhorse SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' + + # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 + # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + SECRET_LENGTH = 32 class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'RepoPath' => repository.path_to_repo, + GL_ID: Gitlab::GlId.gl_id(user), + RepoPath: repository.path_to_repo, } end + def lfs_upload_ok(oid, size) + { + StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + LfsOid: oid, + LfsSize: size, + } + end + + def artifact_upload_ok + { TempPath: ArtifactUploader.artifacts_upload_path } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -81,6 +100,35 @@ module Gitlab path.readable? ? path.read.chomp : 'unknown' end + def secret + @secret ||= begin + bytes = Base64.strict_decode64(File.read(secret_path).chomp) + raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes + end + end + + def write_secret + bytes = SecureRandom.random_bytes(SECRET_LENGTH) + File.open(secret_path, 'w:BINARY', 0600) do |f| + f.chmod(0600) + f.write(Base64.strict_encode64(bytes)) + end + end + + def verify_api_request!(request_headers) + JWT.decode( + request_headers[INTERNAL_API_REQUEST_HEADER], + secret, + true, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + ) + end + + def secret_path + Rails.root.join('.gitlab_workhorse_secret') + end + protected def encode(hash) diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake new file mode 100644 index 00000000000..609dfaa48e3 --- /dev/null +++ b/lib/tasks/haml-lint.rake @@ -0,0 +1,5 @@ +unless Rails.env.production? + require 'haml_lint/rake_task' + + HamlLint::RakeTask.new +end |