diff options
Diffstat (limited to 'lib')
156 files changed, 3669 insertions, 1113 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 7834262d612..7d65145176b 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,5 +54,8 @@ module API mount Keys mount Tags mount Triggers + mount Builds + mount Variables + mount Runners end end diff --git a/lib/api/builds.rb b/lib/api/builds.rb new file mode 100644 index 00000000000..2b104f90aa7 --- /dev/null +++ b/lib/api/builds.rb @@ -0,0 +1,204 @@ +module API + # Projects builds API + class Builds < Grape::API + before { authenticate! } + + resource :projects do + # Get a project builds + # + # Parameters: + # id (required) - The ID of a project + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) + # Example Request: + # GET /projects/:id/builds + get ':id/builds' do + + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end + + # Get builds for a specific commit of a project + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The SHA id of a commit + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) + # Example Request: + # GET /projects/:id/repository/commits/:sha/builds + get ':id/repository/commits/:sha/builds' do + authorize_read_builds! + + commit = user_project.ci_commits.find_by_sha(params[:sha]) + return not_found! unless commit + + builds = commit.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end + + # Get a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/builds/:build_id + get ':id/builds/:build_id' do + authorize_read_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end + + # Download the artifacts file from build + # + # Parameters: + # id (required) - The ID of a build + # token (required) - The build authorization token + # Example Request: + # GET /projects/:id/builds/:build_id/artifacts + get ':id/builds/:build_id/artifacts' do + authorize_read_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + artifacts_file = build.artifacts_file + + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + + return not_found! unless artifacts_file.exists? + + present_file!(artifacts_file.path, artifacts_file.filename) + end + + # Get a trace of a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/build/:build_id/trace + # + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + get ':id/builds/:build_id/trace' do + authorize_read_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary + + trace = build.trace + body trace + end + + # Cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/cancel + post ':id/builds/:build_id/cancel' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + build.cancel + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end + + # Retry a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/retry + post ':id/builds/:build_id/retry' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + return forbidden!('Build is not retryable') unless build.retryable? + + build = Ci::Build.retry(build) + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end + + # Erase build (remove artifacts and build trace) + # + # Parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example Request: + # post /projects/:id/build/:build_id/erase + post ':id/builds/:build_id/erase' do + authorize_update_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + return forbidden!('Build is not erasable!') unless build.erasable? + + build.erase(erased_by: current_user) + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + end + + helpers do + def get_build(id) + user_project.builds.find_by(id: id.to_i) + end + + def filter_builds(builds, scope) + return builds if scope.nil? || scope.empty? + + available_statuses = ::CommitStatus::AVAILABLE_STATUSES + scope = + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? + + builds.where(status: available_statuses && scope) + end + + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + end + end +end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 1162271f5fc..8e74e177ea0 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -18,10 +18,12 @@ module API # Examples: # GET /projects/:id/repository/commits/:sha/statuses get ':id/repository/commits/:sha/statuses' do - authorize! :read_commit_statuses, user_project - sha = params[:sha] - ci_commit = user_project.ci_commit(sha) - not_found! 'Commit' unless ci_commit + authorize!(:read_commit_status, user_project) + + not_found!('Commit') unless user_project.commit(params[:sha]) + ci_commit = user_project.ci_commit(params[:sha]) + return [] unless ci_commit + statuses = ci_commit.statuses statuses = statuses.latest unless parse_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? diff --git a/lib/api/commits.rb b/lib/api/commits.rb index f4efb651eb6..4544a41b1e3 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -48,7 +48,7 @@ module API sha = params[:sha] commit = user_project.commit(sha) not_found! "Commit" unless commit - commit.diffs + commit.diffs.to_a end # Get a commit's comments @@ -90,9 +90,9 @@ module API } if params[:path] && params[:line] && params[:line_type] - commit.diffs.each do |diff| + commit.diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] - lines = Gitlab::Diff::Parser.new.parse(diff.diff.lines.to_a) + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 26e7c956e8f..71197205f34 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -23,12 +23,15 @@ module API end class UserFull < User + expose :last_sign_in_at + expose :confirmed_at expose :email expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project expose :two_factor_enabled + expose :external end class UserLogin < UserFull @@ -49,7 +52,7 @@ module API expose :enable_ssl_verification end - class ForkedFromProject < Grape::Entity + class BasicProjectDetails < Grape::Entity expose :id expose :name, :name_with_namespace expose :path, :path_with_namespace @@ -67,10 +70,12 @@ module API expose :shared_runners_enabled expose :creator_id expose :namespace - expose :forked_from_project, using: Entities::ForkedFromProject, if: lambda{ |project, options| project.forked? } + expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } + expose :public_builds end class ProjectMember < UserBasic @@ -139,7 +144,10 @@ module API class ProjectSnippet < Grape::Entity expose :id, :title, :file_name expose :author, using: Entities::UserBasic - expose :expires_at, :updated_at, :created_at + expose :updated_at, :created_at + + # TODO (rspeicher): Deprecated; remove in 9.0 + expose(:expires_at) { |snippet| nil } end class ProjectEntity < Grape::Entity @@ -174,11 +182,12 @@ module API expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds + expose :merge_status end class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.diffs + compare.diffs(all_diffs: true).to_a end end @@ -238,6 +247,10 @@ module API end end + class ProjectGroupLink < Grape::Entity + expose :id, :project_id, :group_id, :group_access + end + class Namespace < Grape::Entity expose :id, :path, :kind end @@ -292,11 +305,11 @@ module API end expose :diffs, using: Entities::RepoDiff do |compare, options| - compare.diffs + compare.diffs(all_diffs: true).to_a end expose :compare_timeout do |compare, options| - compare.timeout + compare.diffs.overflow? end expose :same, as: :compare_same_ref @@ -365,5 +378,52 @@ module API class TriggerRequest < Grape::Entity expose :id, :variables end + + class Runner < Grape::Entity + expose :id + expose :description + expose :active + expose :is_shared + expose :name + end + + class RunnerDetails < Runner + expose :tag_list + expose :version, :revision, :platform, :architecture + expose :contacted_at + expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? } + expose :projects, with: Entities::BasicProjectDetails do |runner, options| + if options[:current_user].is_admin? + runner.projects + else + options[:current_user].authorized_projects.where(id: runner.projects) + end + end + end + + class BuildArtifactFile < Grape::Entity + expose :filename, :size + end + + class Build < Grape::Entity + expose :id, :status, :stage, :name, :ref, :tag, :coverage + expose :created_at, :started_at, :finished_at + expose :user, with: User + expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } + expose :commit, with: RepoCommit do |repo_obj, _options| + if repo_obj.respond_to?(:commit) + repo_obj.commit.commit_data + end + end + expose :runner, with: Runner + end + + class Trigger < Grape::Entity + expose :token, :created_at, :updated_at, :deleted_at, :last_used + end + + class Variable < Grape::Entity + expose :key, :value + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index 8ad2c1883c7..c1d86f313b0 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -58,9 +58,11 @@ module API commit = user_project.commit(ref) not_found! 'Commit' unless commit - blob = user_project.repository.blob_at(commit.sha, file_path) + repo = user_project.repository + blob = repo.blob_at(commit.sha, file_path) if blob + blob.load_all_data!(repo) status(200) { @@ -72,7 +74,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, - last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id + last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..a72044e8058 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -30,7 +30,7 @@ module API end def sudo_identifier() - identifier ||= params[SUDO_PARAM] ||= env[SUDO_HEADER] + identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers if !!(identifier =~ /^[0-9]+$/) @@ -97,11 +97,9 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) - add_pagination_headers(paginated, per_page) - - paginated + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end end def authenticate! @@ -155,10 +153,11 @@ module API end def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params attrs = {} keys.each do |key| - if params[key].present? or (params.has_key?(key) and params[key] == false) - attrs[key] = params[key] + if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + attrs[key] = params_hash[key] end end ActionController::Parameters.new(attrs).permit! @@ -266,6 +265,10 @@ module API projects = projects.search(params[:search]) end + if params[:visibility].present? + projects = projects.search_by_visibility(params[:visibility]) + end + projects.reorder(project_order_by => project_sort) end @@ -289,12 +292,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) @@ -327,16 +332,36 @@ module API private - def add_pagination_headers(paginated, per_page) + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value links = [] - links << %(<#{request_url}?page=#{paginated.current_page - 1}&per_page=#{per_page}>; rel="prev") unless paginated.first_page? - links << %(<#{request_url}?page=#{paginated.current_page + 1}&per_page=#{per_page}>; rel="next") unless paginated.last_page? - links << %(<#{request_url}?page=1&per_page=#{per_page}>; rel="first") - links << %(<#{request_url}?page=#{paginated.total_pages}&per_page=#{per_page}>; rel="last") - header 'Link', links.join(', ') + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') end def abilities diff --git a/lib/api/internal.rb b/lib/api/internal.rb index e38736fc28b..2200208b946 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -14,6 +14,14 @@ module API # ref - branch name # forced_push - forced_push # + + helpers do + def wiki? + @wiki ||= params[:project].end_with?('.wiki') && + !Project.find_with_namespace(params[:project]) + end + end + post "/allowed" do status 200 @@ -30,13 +38,12 @@ module API # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to # the wiki repository as well. - wiki = project_path.end_with?('.wiki') - project_path.chomp!('.wiki') if wiki + project_path.chomp!('.wiki') if wiki? project = Project.find_with_namespace(project_path) access = - if wiki + if wiki? Gitlab::GitAccessWiki.new(actor, project) else Gitlab::GitAccess.new(actor, project) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 6e7a7672070..fda6f841438 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -3,6 +3,8 @@ module API class Issues < Grape::API before { authenticate! } + helpers ::Gitlab::AkismetHelper + helpers do def filter_issues_state(issues, state) case state @@ -19,6 +21,17 @@ module API def filter_issues_milestone(issues, milestone) issues.includes(:milestone).where('milestones.title' => milestone) end + + def create_spam_log(project, current_user, attrs) + params = attrs.merge({ + source_ip: env['REMOTE_ADDR'], + user_agent: env['HTTP_USER_AGENT'], + noteable_type: 'Issue', + via_api: true + }) + + ::CreateSpamLogService.new(project, current_user, params).execute + end end resource :issues do @@ -69,7 +82,7 @@ module API # GET /projects/:id/issues?milestone=1.0.0&state=closed # GET /issues?iid=42 get ":id/issues" do - issues = user_project.issues + issues = user_project.issues.visible_to_user(current_user) issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? @@ -91,6 +104,7 @@ module API # GET /projects/:id/issues/:issue_id get ":id/issues/:issue_id" do @issue = user_project.issues.find(params[:issue_id]) + not_found! unless can?(current_user, :read_issue, @issue) present @issue, with: Entities::Issue end @@ -114,7 +128,15 @@ module API render_api_error!({ labels: errors }, 400) end - issue = ::Issues::CreateService.new(user_project, current_user, attrs).execute + project = user_project + text = [attrs[:title], attrs[:description]].reject(&:blank?).join("\n") + + if check_for_spam?(project, current_user) && is_spam?(env, current_user, text) + create_spam_log(project, current_user, attrs) + render_api_error!({ error: 'Spam detected' }, 400) + end + + issue = ::Issues::CreateService.new(project, current_user, attrs).execute if issue.valid? # Find or create labels and attach to issue. Labels are valid because diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 3c1c6bda260..c5e5d57ed4d 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -59,55 +59,6 @@ module API present paginate(merge_requests), with: Entities::MergeRequest end - # Show MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id - # - get ":id/merge_request/:merge_request_id" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - - present merge_request, with: Entities::MergeRequest - end - - # Show MR commits - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id/commits - # - get ':id/merge_request/:merge_request_id/commits' do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request.commits, with: Entities::RepoCommit - end - - # Show MR changes - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_request/:merge_request_id/changes - # - get ':id/merge_request/:merge_request_id/changes' do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges - end - # Create MR # # Parameters: @@ -120,6 +71,7 @@ module API # title (required) - Title of MR # description - Description of MR # labels (optional) - Labels for MR as a comma-separated list + # milestone_id (optional) - Milestone ID # # Example: # POST /projects/:id/merge_requests @@ -127,7 +79,7 @@ module API post ":id/merge_requests" do authorize! :create_merge_request, user_project required_attributes! [:source_branch, :target_branch, :title] - attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id, :description] + attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id, :description, :milestone_id] # Validate label names in advance if (errors = validate_label_params(params)).any? @@ -148,146 +100,220 @@ module API end end - # Update MR + # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 + # Use "merge_requests/:merge_request_id/..." instead. # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # target_branch - The target branch - # assignee_id - Assignee user ID - # title - Title of MR - # state_event - Status of MR. (close|reopen|merge) - # description - Description of MR - # labels (optional) - Labels for a MR as a comma-separated list - # Example: - # PUT /projects/:id/merge_request/:merge_request_id - # - put ":id/merge_request/:merge_request_id" do - attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :update_merge_request, merge_request - - # Ensure source_branch is not specified - if params[:source_branch].present? - render_api_error!('Source branch cannot be changed', 400) - end - - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end - - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) - - if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end + [":id/merge_request/:merge_request_id", ":id/merge_requests/:merge_request_id"].each do |path| + # Show MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id + # + get path do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request present merge_request, with: Entities::MergeRequest - else - handle_merge_request_errors! merge_request.errors end - end - - # Merge MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # merge_commit_message (optional) - Custom merge commit message - # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible - # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds - # Example: - # PUT /projects/:id/merge_request/:merge_request_id/merge - # - put ":id/merge_request/:merge_request_id/merge" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! if !merge_request.open? || merge_request.work_in_progress? - merge_request.check_if_can_be_merged if merge_request.unchecked? - - render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? - - merge_params = { - commit_message: params[:merge_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] - } + # Show MR commits + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id/commits + # + get "#{path}/commits" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request.commits, with: Entities::RepoCommit + end - if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? - ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) - else - ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + # Show MR changes + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - The ID of MR + # + # Example: + # GET /projects/:id/merge_requests/:merge_request_id/changes + # + get "#{path}/changes" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + authorize! :read_merge_request, merge_request + present merge_request, with: Entities::MergeRequestChanges end - present merge_request, with: Entities::MergeRequest - end + # Update MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # target_branch - The target branch + # assignee_id - Assignee user ID + # title - Title of MR + # state_event - Status of MR. (close|reopen|merge) + # description - Description of MR + # labels (optional) - Labels for a MR as a comma-separated list + # milestone_id (optional) - Milestone ID + # Example: + # PUT /projects/:id/merge_requests/:merge_request_id + # + put path do + attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description, :milestone_id] + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + authorize! :update_merge_request, merge_request + + # Ensure source_branch is not specified + if params[:source_branch].present? + render_api_error!('Source branch cannot be changed', 400) + end - # Cancel Merge if Merge When build succeeds is enabled - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # - post ":id/merge_request/:merge_request_id/cancel_merge_when_build_succeeds" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + # Validate label names in advance + if (errors = validate_label_params(params)).any? + render_api_error!({ labels: errors }, 400) + end - unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) - ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) - end + if merge_request.valid? + # Find or create labels and attach to issue + unless params[:labels].nil? + merge_request.remove_labels + merge_request.add_labels_by_names(params[:labels].split(",")) + end - # Get a merge request's comments - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_request/:merge_request_id/comments - # - get ":id/merge_request/:merge_request_id/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + present merge_request, with: Entities::MergeRequest + else + handle_merge_request_errors! merge_request.errors + end + end - authorize! :read_merge_request, merge_request + # Merge MR + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # merge_commit_message (optional) - Custom merge commit message + # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible + # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds + # Example: + # PUT /projects/:id/merge_requests/:merge_request_id/merge + # + put "#{path}/merge" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) + not_allowed! if !merge_request.open? || merge_request.work_in_progress? + + merge_request.check_if_can_be_merged + + render_api_error!('Branch cannot be merged', 406) unless merge_request.can_be_merged? + + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.ci_commit && merge_request.ci_commit.active? + ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) + else + ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). + execute(merge_request) + end - present paginate(merge_request.notes.fresh), with: Entities::MRNote - end + present merge_request, with: Entities::MergeRequest + end - # Post comment to merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # note (required) - Text of comment - # Examples: - # POST /projects/:id/merge_request/:merge_request_id/comments - # - post ":id/merge_request/:merge_request_id/comments" do - required_attributes! [:note] + # Cancel Merge if Merge When build succeeds is enabled + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # + post "#{path}/cancel_merge_when_build_succeeds" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) - authorize! :create_note, merge_request + ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) + end - opts = { - note: params[:note], - noteable_type: 'MergeRequest', - noteable_id: merge_request.id - } + # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. + # Use GET "/projects/:id/merge_requests/:merge_request_id/notes" instead + # + # Get a merge request's comments + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # Examples: + # GET /projects/:id/merge_requests/:merge_request_id/comments + # + get "#{path}/comments" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + + present paginate(merge_request.notes.fresh), with: Entities::MRNote + end - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. + # Use POST "/projects/:id/merge_requests/:merge_request_id/notes" instead + # + # Post comment to merge request + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # note (required) - Text of comment + # Examples: + # POST /projects/:id/merge_requests/:merge_request_id/comments + # + post "#{path}/comments" do + required_attributes! [:note] + + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: Entities::MRNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end - if note.save - present note, with: Entities::MRNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) + # List issues that will close on merge + # + # Parameters: + # id (required) - The ID of a project + # merge_request_id (required) - ID of MR + # Examples: + # GET /projects/:id/merge_requests/:merge_request_id/closes_issues + get "#{path}/closes_issues" do + merge_request = user_project.merge_requests.find(params[:merge_request_id]) + issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + present paginate(issues), with: Entities::Issue end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 3efdfe2d46e..174473f5371 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -20,7 +20,19 @@ module API # GET /projects/:id/snippets/:noteable_id/notes get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) - present paginate(@noteable.notes), with: Entities::Note + + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(@noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: Entities::Note end # Get a single +noteable+ note @@ -35,7 +47,12 @@ module API get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do @noteable = user_project.send(:"#{noteables_str}").find(params[:"#{noteable_id_str}"]) @note = @noteable.notes.find(params[:note_id]) - present @note, with: Entities::Note + + if @note.cross_reference_not_visible_for?(current_user) + not_found!("Note") + else + present @note, with: Entities::Note + end end # Create a new +noteable+ note diff --git a/lib/api/projects.rb b/lib/api/projects.rb index a9e0960872a..6fcb5261e40 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,7 +3,7 @@ module API class Projects < Grape::API before { authenticate! } - resource :projects do + resource :projects, requirements: { id: /[^\/]+/ } do helpers do def map_public_to_visibility_level(attrs) publik = attrs.delete(:public) @@ -69,7 +69,8 @@ module API # Example Request: # GET /projects/:id get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user + present user_project, with: Entities::ProjectWithAccess, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) end # Get events for a single project @@ -98,6 +99,7 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - 0 by default # import_url (optional) + # public_builds (optional) # Example Request # POST /projects post do @@ -114,11 +116,13 @@ module API :namespace_id, :public, :visibility_level, - :import_url] + :import_url, + :public_builds] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else if @project.errors[:limit_reached].present? error!(@project.errors[:limit_reached], 403) @@ -143,6 +147,7 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) + # public_builds (optional) # Example Request # POST /projects/user/:user_id post "user/:user_id" do @@ -159,11 +164,13 @@ module API :shared_runners_enabled, :public, :visibility_level, - :import_url] + :import_url, + :public_builds] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? - present @project, with: Entities::Project + present @project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @project) else render_validation_error!(@project) end @@ -182,7 +189,8 @@ module API if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else - present @forked_project, with: Entities::Project + present @forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, @forked_project) end end @@ -201,6 +209,7 @@ module API # shared_runners_enabled (optional) # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project + # public_builds (optional) # Example Request # PUT /projects/:id put ':id' do @@ -215,7 +224,8 @@ module API :snippets_enabled, :shared_runners_enabled, :public, - :visibility_level] + :visibility_level, + :public_builds] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -229,7 +239,8 @@ module API if user_project.errors.any? render_validation_error!(user_project) else - present user_project, with: Entities::Project + present user_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) end end @@ -241,7 +252,7 @@ module API # DELETE /projects/:id delete ":id" do authorize! :remove_project, user_project - ::Projects::DestroyService.new(user_project, current_user, {}).execute + ::Projects::DestroyService.new(user_project, current_user, {}).pending_delete! end # Mark this project as forked from another @@ -269,7 +280,7 @@ module API # Remove a forked_from relationship # # Parameters: - # id: (required) - The ID of the project being marked as a fork + # id: (required) - The ID of the project being marked as a fork # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do @@ -278,6 +289,43 @@ module API user_project.forked_project_link.destroy end end + + # Share project with group + # + # Parameters: + # id (required) - The ID of a project + # group_id (required) - The ID of a group + # group_access (required) - Level of permissions for sharing + # + # Example Request: + # POST /projects/:id/share + post ":id/share" do + authorize! :admin_project, user_project + required_attributes! [:group_id, :group_access] + + unless user_project.allowed_to_share_with_group? + return render_api_error!("The project sharing with group is disabled", 400) + end + + link = user_project.project_group_links.new + link.group_id = params[:group_id] + link.group_access = params[:group_access] + if link.save + present link, with: Entities::ProjectGroupLink + else + render_api_error!(link.errors.full_messages.first, 409) + end + end + + # Upload a file + # + # Parameters: + # id: (required) - The ID of the project + # file: (required) - The file to be uploaded + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + # search for projects current_user has access to # # Parameters: diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index d7c48639eba..0d0f0d4616d 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,7 +57,7 @@ module API not_found! "File" unless blob content_type 'text/plain' - present blob.data + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a raw blob contents by blob sha @@ -83,7 +83,7 @@ module API env['api.format'] = :txt content_type blob.mime_type - present blob.data + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a an archive of the repository @@ -98,11 +98,8 @@ module API authorize! :download_code, user_project begin - ArchiveRepositoryService.new( - user_project, - params[:sha], - params[:format] - ).execute + RepositoryArchiveCacheWorker.perform_async + header *Gitlab::Workhorse.send_git_archive(user_project, params[:sha], params[:format]) rescue not_found!('File') end diff --git a/lib/api/runners.rb b/lib/api/runners.rb new file mode 100644 index 00000000000..8ec91485b26 --- /dev/null +++ b/lib/api/runners.rb @@ -0,0 +1,175 @@ +module API + # Runners API + class Runners < Grape::API + before { authenticate! } + + resource :runners do + # Get runners available for user + # + # Example Request: + # GET /runners + get do + runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) + present paginate(runners), with: Entities::Runner + end + + # Get all runners - shared and specific + # + # Example Request: + # GET /runners/all + get 'all' do + authenticated_as_admin! + runners = filter_runners(Ci::Runner.all, params[:scope]) + present paginate(runners), with: Entities::Runner + end + + # Get runner's details + # + # Parameters: + # id (required) - The ID of ther runner + # Example Request: + # GET /runners/:id + get ':id' do + runner = get_runner(params[:id]) + authenticate_show_runner!(runner) + + present runner, with: Entities::RunnerDetails, current_user: current_user + end + + # Update runner's details + # + # Parameters: + # id (required) - The ID of ther runner + # description (optional) - Runner's description + # active (optional) - Runner's status + # tag_list (optional) - Array of tags for runner + # Example Request: + # PUT /runners/:id + put ':id' do + runner = get_runner(params[:id]) + authenticate_update_runner!(runner) + + attrs = attributes_for_keys [:description, :active, :tag_list] + if runner.update(attrs) + present runner, with: Entities::RunnerDetails, current_user: current_user + else + render_validation_error!(runner) + end + end + + # Remove runner + # + # Parameters: + # id (required) - The ID of ther runner + # Example Request: + # DELETE /runners/:id + delete ':id' do + runner = get_runner(params[:id]) + authenticate_delete_runner!(runner) + runner.destroy! + + present runner, with: Entities::Runner + end + end + + resource :projects do + before { authorize_admin_project } + + # Get runners available for project + # + # Example Request: + # GET /projects/:id/runners + get ':id/runners' do + runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) + present paginate(runners), with: Entities::Runner + end + + # Enable runner for project + # + # Parameters: + # id (required) - The ID of the project + # runner_id (required) - The ID of the runner + # Example Request: + # POST /projects/:id/runners/:runner_id + post ':id/runners' do + required_attributes! [:runner_id] + + runner = get_runner(params[:runner_id]) + authenticate_enable_runner!(runner) + Ci::RunnerProject.create(runner: runner, project: user_project) + + present runner, with: Entities::Runner + end + + # Disable project's runner + # + # Parameters: + # id (required) - The ID of the project + # runner_id (required) - The ID of the runner + # Example Request: + # DELETE /projects/:id/runners/:runner_id + delete ':id/runners/:runner_id' do + runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) + not_found!('Runner') unless runner_project + + runner = runner_project.runner + forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 + + runner_project.destroy + + present runner, with: Entities::Runner + end + end + + helpers do + def filter_runners(runners, scope, options = {}) + return runners unless scope.present? + + available_scopes = ::Ci::Runner::AVAILABLE_SCOPES + if options[:without] + available_scopes = available_scopes - options[:without] + end + + if (available_scopes & [scope]).empty? + render_api_error!('Scope contains invalid value', 400) + end + + runners.send(scope) + end + + def get_runner(id) + runner = Ci::Runner.find(id) + not_found!('Runner') unless runner + runner + end + + def authenticate_show_runner!(runner) + return if runner.is_shared || current_user.is_admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def authenticate_update_runner!(runner) + return if current_user.is_admin? + forbidden!("Runner is shared") if runner.is_shared? + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def authenticate_delete_runner!(runner) + return if current_user.is_admin? + forbidden!("Runner is shared") if runner.is_shared? + forbidden!("Runner associated with more than one project") if runner.projects.count > 1 + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def authenticate_enable_runner!(runner) + forbidden!("Runner is shared") if runner.is_shared? + return if current_user.is_admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def user_can_access_runner?(runner) + current_user.ci_authorized_runners.exists?(runner.id) + end + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 47621f443e6..2d8a9e51bb9 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -40,6 +40,27 @@ module API end end + # Delete tag + # + # Parameters: + # id (required) - The ID of a project + # tag_name (required) - The name of the tag + # Example Request: + # DELETE /projects/:id/repository/tags/:tag + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + authorize_push_project + result = DeleteTagService.new(user_project, current_user). + execute(params[:tag_name]) + + if result[:status] == :success + { + tag_name: params[:tag_name] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + # Add release notes to tag # # Parameters: diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 2781f1cf191..d1d07394e92 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -43,6 +43,75 @@ module API render_api_error!(errors, 400) end end + + # Get triggers list + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/triggers + get ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + triggers = user_project.triggers.includes(:trigger_requests) + triggers = paginate(triggers) + + present triggers, with: Entities::Trigger + end + + # Get specific trigger of a project + # + # Parameters: + # id (required) - The ID of a project + # token (required) - The `token` of a trigger + # Example Request: + # GET /projects/:id/triggers/:token + get ':id/triggers/:token' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger + + present trigger, with: Entities::Trigger + end + + # Create trigger + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # POST /projects/:id/triggers + post ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.create + + present trigger, with: Entities::Trigger + end + + # Delete trigger + # + # Parameters: + # id (required) - The ID of a project + # token (required) - The `token` of a trigger + # Example Request: + # DELETE /projects/:id/triggers/:token + delete ':id/triggers/:token' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger + + trigger.destroy + + present trigger, with: Entities::Trigger + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 3400f0713ef..13ab17c6904 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,7 +39,7 @@ module API if current_user.is_admin? present @user, with: Entities::UserFull else - present @user, with: Entities::UserBasic + present @user, with: Entities::User end end @@ -61,19 +61,20 @@ module API # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false # confirm - Require user confirmation - true (default) or false + # external - Flags the user as external - true or false(default) # Example Request: # POST /users post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :can_create_group, :admin, :confirm, :external] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ (/(false|f|no|0)$/i)) user = User.build_user(attrs) user.admin = admin unless admin.nil? user.skip_confirmation! unless confirm - identity_attrs = attributes_for_keys [:provider, :extern_uid] + if identity_attrs.any? user.identities.build(identity_attrs) end @@ -107,12 +108,13 @@ module API # bio - Bio # admin - User is admin - true or false (default) # can_create_group - User can create groups - true or false + # external - Flags the user as external - true or false(default) # Example Request: # PUT /users/:id put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :can_create_group, :admin, :external] user = User.find(params[:id]) not_found!('User') unless user @@ -284,10 +286,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user + if !user + not_found!('User') + elsif !user.ldap_blocked? user.block else - not_found!('User') + forbidden!('LDAP blocked users cannot be modified by the API') end end @@ -299,10 +303,12 @@ module API authenticated_as_admin! user = User.find_by(id: params[:id]) - if user - user.activate - else + if !user not_found!('User') + elsif user.ldap_blocked? + forbidden!('LDAP blocked users cannot be unblocked by the API') + else + user.activate end end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb new file mode 100644 index 00000000000..f6495071a11 --- /dev/null +++ b/lib/api/variables.rb @@ -0,0 +1,95 @@ +module API + # Projects variables API + class Variables < Grape::API + before { authenticate! } + before { authorize! :admin_build, user_project } + + resource :projects do + # Get project variables + # + # Parameters: + # id (required) - The ID of a project + # page (optional) - The page number for pagination + # per_page (optional) - The value of items per page to show + # Example Request: + # GET /projects/:id/variables + get ':id/variables' do + variables = user_project.variables + present paginate(variables), with: Entities::Variable + end + + # Get specific variable of a project + # + # Parameters: + # id (required) - The ID of a project + # key (required) - The `key` of variable + # Example Request: + # GET /projects/:id/variables/:key + get ':id/variables/:key' do + key = params[:key] + variable = user_project.variables.find_by(key: key.to_s) + + return not_found!('Variable') unless variable + + present variable, with: Entities::Variable + end + + # Create a new variable in project + # + # Parameters: + # id (required) - The ID of a project + # key (required) - The key of variable + # value (required) - The value of variable + # Example Request: + # POST /projects/:id/variables + post ':id/variables' do + required_attributes! [:key, :value] + + variable = user_project.variables.create(key: params[:key], value: params[:value]) + + if variable.valid? + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + # Update existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # key (optional) - The `key` of variable + # value (optional) - New value for `value` field of variable + # Example Request: + # PUT /projects/:id/variables/:key + put ':id/variables/:key' do + variable = user_project.variables.find_by(key: params[:key].to_s) + + return not_found!('Variable') unless variable + + attrs = attributes_for_keys [:value] + if variable.update(attrs) + present variable, with: Entities::Variable + else + render_validation_error!(variable) + end + end + + # Delete existing variable of a project + # + # Parameters: + # id (required) - The ID of a project + # key (required) - The ID of a variable + # Example Request: + # DELETE /projects/:id/variables/:key + delete ':id/variables/:key' do + variable = user_project.variables.find_by(key: params[:key].to_s) + + return not_found!('Variable') unless variable + variable.destroy + + present variable, with: Entities::Variable + end + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 099062eeb8b..4962f5e53ce 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,6 +1,9 @@ module Backup class Manager def pack + # Make sure there is a connection + ActiveRecord::Base.connection.reconnect! + # saving additional informations s = {} s[:db_version] = "#{ActiveRecord::Migrator.current_version}" diff --git a/lib/banzai.rb b/lib/banzai.rb index 093382261ae..b467413a7dd 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -7,6 +7,10 @@ module Banzai Renderer.render_result(text, context) end + def self.pre_process(text, context) + Renderer.pre_process(text, context) + end + def self.post_process(html, context) Renderer.post_process(html, context) end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index ba2866e1efa..0257848b6bc 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Common methods for ReferenceFilters that support an optional cross-project # reference. diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index fd4fe024252..905c4c0144e 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' module Banzai module Filter diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 63ad8910c0f..34c38913474 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # Issues, Merge Requests, Snippets, Commits and Commit Ranges share @@ -47,7 +45,17 @@ module Banzai { object_sym => LazyReference.new(object_class, node.attr(data_reference)) } end - delegate :object_class, :object_sym, :references_in, to: :class + def object_class + self.class.object_class + end + + def object_sym + self.class.object_sym + end + + def references_in(*args, &block) + self.class.references_in(*args, &block) + end def find_object(project, id) # Implement in child class @@ -60,28 +68,34 @@ module Banzai end def call - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + if object_class.reference_pattern + # `#123` + replace_text_nodes_matching(object_class.reference_pattern) do |content| + object_link_filter(content, object_class.reference_pattern) + end - # `[Issue](#123)`, which is turned into - # `<a href="#123">Issue</a>` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) + # `[Issue](#123)`, which is turned into + # `<a href="#123">Issue</a>` + replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| + object_link_filter(link, object_class.reference_pattern, link_text: text) + end end - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + if object_class.link_reference_pattern + # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">http://gitlab.example.com/namespace/project/issues/123</a>` + replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| + object_link_filter(text, object_class.link_reference_pattern) + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into + # `<a href="http://gitlab.example.com/namespace/project/issues/123">Issue</a>` + replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| + object_link_filter(link, object_class.link_reference_pattern, link_text: text) + end end + + doc end # Replace references (like `!123` for merge requests) in text with links diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index da4ee80c1b5..856f56fb175 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index e67cd45ab9b..470727ee312 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit range references with links. diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 9e57608b483..713a56ba949 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit references with links. diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 86838e1483c..207437ba7cf 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,5 +1,4 @@ require 'action_controller' -require 'banzai' require 'gitlab_emoji' require 'html/pipeline/filter' @@ -46,7 +45,8 @@ module Banzai private def emoji_url(name) - emoji_path = "emoji/#{emoji_filename(name)}" + emoji_path = emoji_filename(name) + if context[:asset_host] # Asset host is specified. url_to_image(emoji_path) diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 6136e73c096..edc26386903 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces external issue tracker references with links. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index ac87b9820af..8d368f3b9e7 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb new file mode 100644 index 00000000000..f31f921903b --- /dev/null +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -0,0 +1,174 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + # HTML Filter for parsing Gollum's tags in HTML. It's only parses the + # following tags: + # + # - Link to internal pages: + # + # * [[Bug Reports]] + # * [[How to Contribute|Contributing]] + # + # - Link to external resources: + # + # * [[http://en.wikipedia.org/wiki/Git_(software)]] + # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]] + # + # - Link internal images, the special attributes will be ignored: + # + # * [[images/logo.png]] + # * [[images/logo.png|alt=Logo]] + # + # - Link external images, the special attributes will be ignored: + # + # * [[http://example.com/images/logo.png]] + # * [[http://example.com/images/logo.png|alt=Logo]] + # + # - Insert a Table of Contents list: + # + # * [[_TOC_]] + # + # Based on Gollum::Filter::Tags + # + # Context options: + # :project_wiki (required) - Current project wiki. + # + class GollumTagsFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match tags content that should be parsed in HTML. + # + # Gollum's tags have been made to resemble the tags of other markups, + # especially MediaWiki. The basic syntax is: + # + # [[tag]] + # + # Some tags will accept attributes which are separated by pipe + # symbols.Some attributes must precede the tag and some must follow it: + # + # [[prefix-attribute|tag]] + # [[tag|suffix-attribute]] + # + # See https://github.com/gollum/gollum/wiki + # + # Rubular: http://rubular.com/r/7dQnE5CUCH + TAGS_PATTERN = %r{\[\[(.+?)\]\]}.freeze + + # Pattern to match allowed image extensions + ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i.freeze + + def call + search_text_nodes(doc).each do |node| + # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running + # before this one, it will be converted into `[[<em>TOC</em>]]`, so it + # needs special-case handling + if toc_tag?(node) + process_toc_tag(node) + else + content = node.content + + next unless content =~ TAGS_PATTERN + + html = process_tag($1) + + if html && html != node.content + node.replace(html) + end + end + end + + doc + end + + private + + # Replace an entire `[[<em>TOC</em>]]` node with the result generated by + # TableOfContentsFilter + def process_toc_tag(node) + node.parent.parent.replace(result[:toc].presence || '') + end + + # Process a single tag into its final HTML form. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML version of the tag. + def process_tag(tag) + parts = tag.split('|') + + return if parts.size.zero? + + process_image_tag(parts) || process_page_link_tag(parts) + end + + # Attempt to process the tag as an image tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid image tag or nil + # if it is not. + def process_image_tag(parts) + content = parts[0].strip + + return unless image?(content) + + if url?(content) + path = content + elsif file = project_wiki.find_file(content) + path = ::File.join project_wiki_base_path, file.path + end + + if path + content_tag(:img, nil, src: path) + end + end + + def toc_tag?(node) + node.content == 'TOC' && + node.parent.name == 'em' && + node.parent.parent.text == '[[TOC]]' + end + + def image?(path) + path =~ ALLOWED_IMAGE_EXTENSIONS + end + + def url?(path) + path.start_with?(*%w(http https)) + end + + # Attempt to process the tag as a page link tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid page link tag or nil + # if it is not. + def process_page_link_tag(parts) + if parts.size == 1 + url = parts[0].strip + else + name, url = *parts.compact.map(&:strip) + end + + content_tag(:a, name || url, href: url) + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + + # Ensure that a :project_wiki key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project_wiki + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 51180cb901a..2732e0b5145 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces issue references with links. References to @@ -11,6 +9,11 @@ module Banzai Issue end + def self.user_can_see_reference?(user, node, context) + issue = Issue.find(node.attr('data-issue')) rescue nil + Ability.abilities.allowed?(user, :read_issue, issue) + end + def find_object(project, id) project.get_issue(id) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a3a7a23c1e6..8147e5ed3c7 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,80 +1,47 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces label references with links. - class LabelReferenceFilter < ReferenceFilter - # Public: Find label references in text - # - # LabelReferenceFilter.references_in(text) do |match, id, name| - # "<a href=...>#{Label.find(id)}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, an optional Integer label ID, and an optional - # String label name. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Label.reference_pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name] - end + class LabelReferenceFilter < AbstractReferenceFilter + def self.object_class + Label end - def self.referenced_by(node) - { label: LazyReference.new(Label, node.attr("data-label")) } + def find_object(project, id) + project.labels.find(id) end - def call - replace_text_nodes_matching(Label.reference_pattern) do |content| - label_link_filter(content) - end - - replace_link_nodes_with_href(Label.reference_pattern) do |link, text| - label_link_filter(link, link_text: text) + def self.references_in(text, pattern = Label.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ end end - # Replace label references in text with links to the label specified. - # - # text - String text to replace references in. - # - # Returns a String with label references replaced with links. All links - # have `gfm` and `gfm-label` class names attached for styling. - def label_link_filter(text, link_text: nil) - project = context[:project] - - self.class.references_in(text) do |match, id, name| - params = label_params(id, name) - - if label = project.labels.find_by(params) - url = url_for_label(project, label) - klass = reference_class(:label) - data = data_attribute( - original: link_text || match, - project: project.id, - label: label.id - ) + def references_in(text, pattern = Label.reference_pattern) + text.gsub(pattern) do |match| + project = project_from_ref($~[:project]) + params = label_params($~[:label_id].to_i, $~[:label_name]) + label = project.labels.find_by(params) - text = link_text || render_colored_label(label) - - %(<a href="#{url}" #{data} - class="#{klass}">#{escape_once(text)}</a>) + if label + yield match, label.id, $~[:project], $~ else match end end end - def url_for_label(project, label) + def url_for_object(label, project) h = Gitlab::Application.routes.url_helpers - h.namespace_project_issues_url( project.namespace, project, label_name: label.name, - only_path: context[:only_path]) + h.namespace_project_issues_url(project.namespace, project, label_name: label.name, + only_path: context[:only_path]) end - def render_colored_label(label) - LabelsHelper.render_colored_label(label) + def object_link_text(object, matches) + if context[:project] == object.project + LabelsHelper.render_colored_label(object) + else + LabelsHelper.render_colored_cross_project_label(object) + end end # Parameters to pass to `Label.find_by` based on the given arguments diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index d09cf41df39..0659fed1419 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 755b946a34b..57c71708992 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces merge request references with links. References diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb new file mode 100644 index 00000000000..e88b27c1fae --- /dev/null +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -0,0 +1,22 @@ +require 'banzai' + +module Banzai + module Filter + # HTML filter that replaces milestone references with links. + class MilestoneReferenceFilter < AbstractReferenceFilter + def self.object_class + Milestone + end + + def find_object(project, id) + project.milestones.find_by(iid: id) + end + + def url_for_object(issue, project) + h = Gitlab::Application.routes.url_helpers + h.namespace_project_milestone_url(project.namespace, project, milestone, + only_path: context[:only_path]) + end + end + end +end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index f01a32b5ae5..7141ed7c9bd 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai @@ -10,7 +9,7 @@ module Banzai # class RedactorFilter < HTML::Pipeline::Filter def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| unless user_can_see_reference?(node) # The reference should be replaced by the original text, # which is not always the same as the rendered text. diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 8ca05ace88c..3637b1bac94 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' require 'html/pipeline/filter' module Banzai @@ -124,7 +123,7 @@ module Banzai def replace_link_nodes_with_text(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -133,7 +132,8 @@ module Banzai next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? # Ignore ending punctionation like periods or commas next unless link == text && text =~ /\A#{pattern}/ @@ -162,7 +162,7 @@ module Banzai def replace_link_nodes_with_href(pattern) return doc if project.nil? - doc.search('a').each do |node| + doc.xpath('descendant-or-self::a').each do |node| klass = node.attr('class') next if klass && klass.include?('gfm') @@ -170,7 +170,8 @@ module Banzai text = node.text next unless link && text - link = URI.decode(link) + link = CGI.unescape(link) + next unless link.force_encoding('UTF-8').valid_encoding? next unless link && link =~ /\A#{pattern}\z/ html = yield link, text diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 12412ff7ea9..86d484feb90 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai @@ -16,7 +15,7 @@ module Banzai end def call - doc.css('a.gfm').each do |node| + Querying.css(doc, 'a.gfm').each do |node| gather_references(node) end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5a081125f21..41380627d39 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' @@ -91,7 +90,7 @@ module Banzai parts = request_path.split('/') parts.pop if path_type(request_path) != 'tree' - while parts.length > 1 && path.start_with?('../') + while path.start_with?('../') parts.pop path.sub!('../', '') end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d03e3ae4b3c..e8011519608 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' @@ -8,15 +7,10 @@ module Banzai # # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + def whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - if context[:inline_sanitization] - whitelist = LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - else - whitelist = super - end + whitelist = super customize_whitelist(whitelist) @@ -44,11 +38,15 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow abbr elements with title attribute + whitelist[:elements].push('abbr') + whitelist[:attributes]['abbr'] = %w(title) + # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') - # ...but then remove links with the `javascript` protocol - whitelist[:transformers].push(remove_javascript_links) + # ...but then remove links with unsafe protocols + whitelist[:transformers].push(remove_unsafe_links) # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -59,14 +57,19 @@ module Banzai whitelist end - def remove_javascript_links + def remove_unsafe_links lambda do |env| node = env[:node] return unless node.name == 'a' return unless node.has_attribute?('href') - if node['href'].start_with?('javascript', ':javascript') + begin + uri = Addressable::URI.parse(node['href']) + uri.scheme.strip! if uri.scheme + + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + rescue Addressable::URI::InvalidURIError node.remove_attribute('href') end end diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 1ad5df96f85..c870a42f741 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces snippet references with links. References to diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index c889cc1e97c..8c5855e5ffc 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 9b3e67206d5..4056dcd6d64 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index bdf7c2ebdfc..66608c9859c 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'task_list/filter' module Banzai @@ -12,13 +11,18 @@ module Banzai # # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class(node, *new_class_names) + def add_css_class_with_fix(node, *new_class_names) if new_class_names.include?('task-list') - super if node.children.any? { |c| c['class'] == 'task-list-item' } - else - super + # Don't add class to all lists + return + elsif new_class_names.include?('task-list-item') + add_css_class_without_fix(node.parent, 'task-list') end + + add_css_class_without_fix(node, *new_class_names) end + + alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 1a1d0aad8ca..f642aee0967 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 964ab60f614..24f16f8b547 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces user or group references with links. diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb new file mode 100644 index 00000000000..e4e2f3f228d --- /dev/null +++ b/lib/banzai/filter/yaml_front_matter_filter.rb @@ -0,0 +1,28 @@ +require 'html/pipeline/filter' +require 'yaml' + +module Banzai + module Filter + class YamlFrontMatterFilter < HTML::Pipeline::Filter + DELIM = '---'.freeze + + # Hat-tip to Middleman: https://git.io/v2e0z + PATTERN = %r{ + \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)? + (?<start>#{DELIM})[ ]*\r?\n + (?<frontmatter>.*?)[ ]*\r?\n? + ^(?<stop>#{DELIM})[ ]*\r?\n? + \r?\n? + (?<content>.*) + }mx.freeze + + def call + match = PATTERN.match(html) + + return html unless match + + "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}" + end + end + end +end diff --git a/lib/banzai/filter_array.rb b/lib/banzai/filter_array.rb new file mode 100644 index 00000000000..77835a14027 --- /dev/null +++ b/lib/banzai/filter_array.rb @@ -0,0 +1,27 @@ +module Banzai + class FilterArray < Array + # Insert a value immediately after another value + # + # If the preceding value does not exist, the new value is added to the end + # of the Array. + def insert_after(after_value, value) + i = index(after_value) || length - 1 + + insert(i + 1, value) + end + + # Insert a value immediately before another value + # + # If the succeeding value does not exist, the new value is added to the + # beginning of the Array. + def insert_before(before_value, value) + i = index(before_value) || -1 + + if i < 0 + unshift(value) + else + insert(i, value) + end + end + end +end diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb index 073ec5d9801..1095b4debc7 100644 --- a/lib/banzai/lazy_reference.rb +++ b/lib/banzai/lazy_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai class LazyReference def self.load(refs) diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb index 4e017809d9d..142a9962eb1 100644 --- a/lib/banzai/pipeline.rb +++ b/lib/banzai/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline def self.[](name) diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb deleted file mode 100644 index 5e76a817be5..00000000000 --- a/lib/banzai/pipeline/asciidoc_pipeline.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'banzai' - -module Banzai - module Pipeline - class AsciidocPipeline < BasePipeline - def self.filters - [ - Filter::RelativeLinkFilter - ] - end - end - end -end diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 957f352aec5..9694e4bc23f 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AtomPipeline < FullPipeline diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index cd30009e5c0..f60966c3c0f 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,11 +1,10 @@ -require 'banzai' require 'html/pipeline' module Banzai module Pipeline class BasePipeline def self.filters - [] + FilterArray[] end def self.transform_context(context) diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb new file mode 100644 index 00000000000..adc09c8afbd --- /dev/null +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -0,0 +1,16 @@ +module Banzai + module Pipeline + class BroadcastMessagePipeline < DescriptionPipeline + def self.filters + @filters ||= FilterArray[ + Filter::MarkdownFilter, + Filter::SanitizationFilter, + + Filter::EmojiFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb index f3bf1809d18..60190f8d9dd 100644 --- a/lib/banzai/pipeline/combined_pipeline.rb +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline module CombinedPipeline @@ -12,7 +10,7 @@ module Banzai end def self.filters - pipelines.flat_map(&:filters) + FilterArray.new(pipelines.flat_map(&:filters)) end def self.transform_context(context) diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 94c2cb165a5..f2395867658 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,14 +1,23 @@ -require 'banzai' - module Banzai module Pipeline class DescriptionPipeline < FullPipeline def self.transform_context(context) super(context).merge( # SanitizationFilter - inline_sanitization: true + whitelist: whitelist ) end + + private + + def self.whitelist + # Descriptions are more heavily sanitized, allowing only a few elements. + # See http://git.io/vkuAN + whitelist = Banzai::Filter::SanitizationFilter::LIMITED + whitelist[:elements] -= %w(pre code img ol ul li) + + whitelist + end end end end diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 14356145a35..e47c384afc1 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class EmailPipeline < FullPipeline diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index 72395a5d50e..d47ddfda4be 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 38750b55ec7..8cd4b50e65a 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,10 +1,8 @@ -require 'banzai' - module Banzai module Pipeline class GfmPipeline < BasePipeline def self.filters - @filters ||= [ + @filters ||= FilterArray[ Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, @@ -22,6 +20,7 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, + Filter::MilestoneReferenceFilter, Filter::TaskListFilter ] diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb index 89335143852..7890f20f716 100644 --- a/lib/banzai/pipeline/note_pipeline.rb +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class NotePipeline < FullPipeline diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 998fd75daa2..3f45db21869 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -1,10 +1,8 @@ -require 'banzai' - module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline def self.filters - [ + FilterArray[ Filter::MarkdownFilter ] end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 148f24b6ce1..ecff094b1e5 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -1,10 +1,8 @@ -require 'banzai' - module Banzai module Pipeline class PostProcessPipeline < BasePipeline def self.filters - [ + FilterArray[ Filter::RelativeLinkFilter, Filter::RedactorFilter ] diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb new file mode 100644 index 00000000000..50dc978b452 --- /dev/null +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -0,0 +1,17 @@ +module Banzai + module Pipeline + class PreProcessPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::YamlFrontMatterFilter + ] + end + + def self.transform_context(context) + context.merge( + pre_process: true + ) + end + end + end +end diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb index 4f9bc9fcccc..919998380e4 100644 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -1,10 +1,8 @@ -require 'banzai' - module Banzai module Pipeline class ReferenceExtractionPipeline < BasePipeline def self.filters - [ + FilterArray[ Filter::ReferenceGathererFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 6725c9039a9..ba2555df98d 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -1,9 +1,23 @@ -require 'banzai' - module Banzai module Pipeline class SingleLinePipeline < GfmPipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + + Filter::EmojiFilter, + Filter::AutolinkFilter, + Filter::ExternalLinkFilter, + Filter::UserReferenceFilter, + Filter::IssueReferenceFilter, + Filter::ExternalIssueReferenceFilter, + Filter::MergeRequestReferenceFilter, + Filter::SnippetReferenceFilter, + Filter::CommitRangeReferenceFilter, + Filter::CommitReferenceFilter, + ] + end end end end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb new file mode 100644 index 00000000000..9b4ff0f0f80 --- /dev/null +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -0,0 +1,12 @@ +require 'banzai' + +module Banzai + module Pipeline + class WikiPipeline < FullPipeline + def self.filters + @filters ||= super.insert_after(Filter::TableOfContentsFilter, + Filter::GollumTagsFilter) + end + end + end +end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb new file mode 100644 index 00000000000..1e1b51e683e --- /dev/null +++ b/lib/banzai/querying.rb @@ -0,0 +1,18 @@ +module Banzai + module Querying + # Searches a Nokogiri document using a CSS query, optionally optimizing it + # whenever possible. + # + # document - A document/element to search. + # query - The CSS query to use. + # + # Returns a Nokogiri::XML::NodeSet. + def self.css(document, query) + # When using "a.foo" Nokogiri compiles this to "//a[...]" but + # "descendant::a[...]" is quite a bit faster and achieves the same result. + xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + + document.xpath(xpath) + end + end +end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 2c197d31898..f4079538ec5 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 115ae914524..ae714c87dc5 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - CACHE_ENABLED = false - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -20,7 +18,7 @@ module Banzai cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) - if cache_key && CACHE_ENABLED + if cache_key Rails.cache.fetch(cache_key) do cacheless_render(text, context) end @@ -33,6 +31,12 @@ module Banzai Pipeline[context[:pipeline]].call(text, context) end + def self.pre_process(text, context) + pipeline = Pipeline[:pre_process] + + pipeline.to_html(text, context) + end + # Perform post-processing on an HTML String # # This method is used to perform state-dependent changes to a String of diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 5c347e432b4..4e85d2c3c74 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -25,7 +25,7 @@ module Ci format :json - helpers Helpers + helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..2e9a5d311f9 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -13,14 +13,14 @@ module Ci post "register" do authenticate_runner! update_runner_last_contact + update_runner_info required_attributes! [:token] not_found! unless current_runner.active? build = Ci::RegisterBuildService.new.execute(current_runner) if build - update_runner_info - present build, with: Entities::Build + present build, with: Entities::BuildDetails else not_found! end @@ -38,6 +38,8 @@ module Ci authenticate_runner! update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) + forbidden!('Build has been erased!') if build.erased? + build.update_attributes(trace: params[:trace]) if params[:trace] case params[:state].to_s @@ -78,11 +80,13 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - filename (generated by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -96,13 +100,21 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? + forbidden!('Build is not running!') unless build.running? + forbidden!('Build has been erased!') if build.erased? + + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size - if build.update_attributes(artifacts_file: file) - present build, with: Entities::Build + build.artifacts_file = artifacts + build.artifacts_metadata = metadata + + if build.save + present(build, with: Entities::BuildDetails) else render_validation_error!(build) end @@ -134,7 +146,7 @@ module Ci present_file!(artifacts_file.path, artifacts_file.filename) end - # Remove the artifacts file from build + # Remove the artifacts file from build - Runners only # # Parameters: # id (required) - The ID of a build @@ -147,7 +159,9 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) + build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index e4ac0545ea2..b25e0e573a8 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -16,10 +16,19 @@ module Ci end class Build < Grape::Entity - expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, - :before_sha, :allow_git_fetch, :project_name - + expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage + expose :project_id + expose :project_name + expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + end + + class BuildDetails < Build + expose :commands + expose :repo_url + expose :before_sha + expose :allow_git_fetch + expose :token expose :options do |model| model.options @@ -30,7 +39,7 @@ module Ci end expose :variables - expose :artifacts_file, using: ArtifactFile + expose :depends_on_builds, using: Build end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 1c91204e98c..199d62d9b8a 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -34,10 +34,14 @@ module Ci @runner ||= Runner.find_by_token(params[:token].to_s) end - def update_runner_info + def get_runner_version_from_params return unless params["info"].present? - info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) - current_runner.update(info) + attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) + end + + def update_runner_info + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? end def max_artifacts_size diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index bfc14fe7a6b..192b1d18a51 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -47,6 +47,7 @@ module Ci return forbidden! unless runner if runner.id + runner.update(get_runner_version_from_params) present runner, with: Entities::Runner else not_found! diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index bcdfd38d292..c89e1b51019 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,12 +5,14 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, + :allow_failure, :type, :stage, :when, :artifacts, :cache, + :dependencies] attr_reader :before_script, :image, :services, :variables, :path, :cache def initialize(config, path = nil) - @config = YAML.safe_load(config, [Symbol]) + @config = YAML.safe_load(config, [Symbol], [], true) @path = path unless @config.is_a? Hash @@ -60,6 +62,7 @@ module Ci @jobs = {} @config.each do |key, job| + next if key.to_s.start_with?('.') stage = job[:stage] || job[:type] || DEFAULT_STAGE @jobs[key] = { stage: stage }.merge(job) end @@ -81,6 +84,7 @@ module Ci services: job[:services] || @services, artifacts: job[:artifacts], cache: job[:cache] || @cache, + dependencies: job[:dependencies], }.compact } end @@ -115,6 +119,10 @@ module Ci end if @cache + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" + end + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) raise ValidationError, "cache:untracked parameter should be an boolean" end @@ -139,6 +147,7 @@ module Ci validate_job_stage!(name, job) if job[:stage] validate_job_cache!(name, job) if job[:cache] validate_job_artifacts!(name, job) if job[:artifacts] + validate_job_dependencies!(name, job) if job[:dependencies] end private @@ -198,6 +207,10 @@ module Ci end def validate_job_cache!(name, job) + if job[:cache][:key] && !validate_string(job[:cache][:key]) + raise ValidationError, "#{name} job: cache:key parameter should be a string" + end + if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked]) raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean" end @@ -208,6 +221,10 @@ module Ci end def validate_job_artifacts!(name, job) + if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) + raise ValidationError, "#{name} job: artifacts:name parameter should be a string" + end + if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked]) raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean" end @@ -217,6 +234,22 @@ module Ci end end + def validate_job_dependencies!(name, job) + if !validate_array_of_strings(job[:dependencies]) + raise ValidationError, "#{name} job: dependencies parameter should be an array of strings" + end + + stage_index = stages.index(job[:stage]) + + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency] + + unless stages.index(@jobs[dependency][:stage]) < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end + end + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end diff --git a/lib/ci/status.rb b/lib/ci/status.rb index c02b3b8f3e4..3fb1fe29494 100644 --- a/lib/ci/status.rb +++ b/lib/ci/status.rb @@ -1,11 +1,9 @@ module Ci class Status def self.get_status(statuses) - statuses.reject! { |status| status.try(&:allow_failure?) } - if statuses.none? 'skipped' - elsif statuses.all?(&:success?) + elsif statuses.all? { |status| status.success? || status.ignored? } 'success' elsif statuses.all?(&:pending?) 'pending' diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb new file mode 100644 index 00000000000..b366c89889e --- /dev/null +++ b/lib/gitlab/akismet_helper.rb @@ -0,0 +1,39 @@ +module Gitlab + module AkismetHelper + def akismet_enabled? + current_application_settings.akismet_enabled + end + + def akismet_client + @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, + Gitlab.config.gitlab.url) + end + + def check_for_spam?(project, user) + akismet_enabled? && !project.team.member?(user) + end + + def is_spam?(environment, user, text) + client = akismet_client + ip_address = environment['REMOTE_ADDR'] + user_agent = environment['HTTP_USER_AGENT'] + + params = { + type: 'comment', + text: text, + created_at: DateTime.now, + author: user.name, + author_email: user.email, + referrer: environment['HTTP_REFERER'], + } + + begin + is_spam, is_blatant = client.check(ip_address, user_agent, params) + is_spam || is_blatant + rescue => e + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + false + end + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index b203b9d70e4..0b9c2e730f9 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -31,9 +31,7 @@ module Gitlab html = ::Asciidoctor.convert(input, asciidoc_opts) - if context[:project] - html = Banzai.render(html, context.merge(pipeline: :asciidoc)) - end + html = Banzai.post_process(html, context) html.html_safe end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 459e3d6bcdb..b9bb6e76081 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -36,7 +36,7 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) raise Error, output unless status.zero? true end @@ -47,7 +47,7 @@ module Gitlab # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new.git") + # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") # def mv_repository(path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', @@ -150,6 +150,18 @@ module Gitlab "#{path}.git", tag_name]) end + # Gc repository + # + # path - project path with namespace + # + # Ex. + # gc("gitlab/gitlab-ci") + # + def gc(path) + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', + "#{path}.git"]) + end + # Add new key to gitlab-shell # # Ex. diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 2355b3c6ddc..46e51a4bf6d 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -13,12 +13,36 @@ module Gitlab end def execute - project_identifier = project.import_source + import_issues if has_issues? - return true unless client.project(project_identifier)["has_issues"] + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error.new, e.message + ensure + Gitlab::BitbucketImport::KeyDeleter.new(project).execute + end - #Issues && Comments - issues = client.issues(project_identifier) + private + + def gl_user_id(project, bitbucket_id) + if bitbucket_id + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + else + project.creator_id + end + end + + def identifier + project.import_source + end + + def has_issues? + client.project(identifier)["has_issues"] + end + + def import_issues + issues = client.issues(identifier) issues.each do |issue| body = '' @@ -33,7 +57,7 @@ module Gitlab body = @formatter.author_line(author) body += issue["content"] - comments = client.issue_comments(project_identifier, issue["local_id"]) + comments = client.issue_comments(identifier, issue["local_id"]) if comments.any? body += @formatter.comments_header @@ -52,24 +76,13 @@ module Gitlab project.issues.create!( description: body, title: issue["title"], - state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', + state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', author_id: gl_user_id(project, reporter) ) end - - true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end - - private - - def gl_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end - end end end end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb new file mode 100644 index 00000000000..997a22779a0 --- /dev/null +++ b/lib/gitlab/blame.rb @@ -0,0 +1,55 @@ +module Gitlab + class Blame + attr_accessor :blob, :commit + + def initialize(blob, commit) + @blob = blob + @commit = commit + end + + def groups(highlight: true) + prev_sha = nil + groups = [] + current_group = nil + + i = 0 + blame.each do |commit, line| + commit = Commit.new(commit, project) + + sha = commit.sha + if prev_sha != sha + groups << current_group if current_group + current_group = { commit: commit, lines: [] } + end + + line = highlighted_lines[i].html_safe if highlight + current_group[:lines] << line + + prev_sha = sha + i += 1 + end + groups << current_group if current_group + + groups + end + + private + + def blame + @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path) + end + + def highlighted_lines + @blob.load_all_data!(repository) + @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + end + + def project + commit.project + end + + def repository + project.repository + end + end +end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 86bfa0a4378..34e949130da 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -23,6 +23,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_allow_failure: build.allow_failure, # TODO: do we still need it? project_id: project.id, diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..f2020c82d40 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,111 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + class ParserError < StandardError; end + + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + + attr_reader :file, :path, :full_version + + def initialize(file, path, **opts) + @file, @path, @opts = file, path, opts + @full_version = read_version + end + + def version + @full_version.match(VERSION_PATTERN)[1] + end + + def errors + gzip do |gz| + read_string(gz) # version + errors = read_string(gz) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end + end + end + + def find_entries! + gzip do |gz| + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) + end + end + + def to_entry + entries = find_entries! + Entry.new(@path, entries) + end + + private + + def match_entries(gz) + entries = {} + + child_pattern = '[^/]*/?$' unless @opts[:recursive] + match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ + + until gz.eof? do + begin + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + + next unless path.valid_encoding? && meta.valid_encoding? + next unless path =~ match_pattern + next if path =~ INVALID_PATH_PATTERN + + entries[path] = JSON.parse(meta, symbolize_names: true) + rescue JSON::ParserError, Encoding::CompatibilityError + next + end + end + + entries + end + + def read_version + gzip do |gz| + version_string = read_string(gz) + + unless version_string + raise ParserError, 'Artifacts metadata file empty!' + end + + unless version_string =~ VERSION_PATTERN + raise ParserError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_uint32(gz) + return nil unless string_size + gz.read(string_size) + end + + def gzip(&block) + Zlib::GzipReader.open(@file, &block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..7f4c750b6fd --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,126 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entries + attr_accessor :name + + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') + @entries = entries + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path) + end + + def empty? + children.empty? + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end + end + end + end +end diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb deleted file mode 100644 index 0d696a1ee28..00000000000 --- a/lib/gitlab/compare_result.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - class CompareResult - attr_reader :commits, :diffs - - def initialize(compare, diff_options = {}) - @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) - end - end -end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7a86c09158e..761b63e98f6 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -4,11 +4,14 @@ module Gitlab key = :current_application_settings RequestStore.store[key] ||= begin + settings = nil + if connect_to_db? - ApplicationSetting.current || ApplicationSetting.create_from_defaults - else - fake_application_settings + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end end @@ -18,29 +21,36 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], + twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], sign_in_text: Settings.extra['sign_in_text'], restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'], + default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], + default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], + import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], + require_two_factor_authentication: false, + two_factor_grace_period: 48, + akismet_enabled: false ) end private def connect_to_db? - use_db = if ENV['USE_DB'] == "false" - false - else - true - end - - use_db && ActiveRecord::Base.connection.active? && - !ActiveRecord::Migrator.needs_migration? && - ActiveRecord::Base.connection.table_exists?('application_settings') + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false + + ENV['USE_DB'] != 'false' && + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + + rescue ActiveRecord::NoDatabaseError + false end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index de77a6fbff1..6f9da69983a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,16 +1,23 @@ module Gitlab module Database + def self.adapter_name + connection.adapter_name + end + def self.mysql? - ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' + adapter_name.downcase == 'mysql2' end def self.postgresql? - ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' + adapter_name.downcase == 'postgresql' + end + + def self.version + database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end def true_value - case ActiveRecord::Base.connection.adapter_name.downcase - when 'postgresql' + if Gitlab::Database.postgresql? "'t'" else 1 @@ -18,12 +25,27 @@ module Gitlab end def false_value - case ActiveRecord::Base.connection.adapter_name.downcase - when 'postgresql' + if Gitlab::Database.postgresql? "'f'" else 0 end end + + private + + def self.connection + ActiveRecord::Base.connection + end + + def self.database_version + row = connection.execute("SELECT VERSION()").first + + if postgresql? + row['version'] + else + row.first + end + end end end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb new file mode 100644 index 00000000000..a78fde9d782 --- /dev/null +++ b/lib/gitlab/devise_failure.rb @@ -0,0 +1,23 @@ +module Gitlab + class DeviseFailure < Devise::FailureApp + protected + + # Override `Devise::FailureApp#request_format` to handle a special case + # + # This tells Devise to handle an unauthenticated `.zip` request as an HTML + # request (i.e., redirect to sign in). + # + # Otherwise, Devise would respond with a 401 Unauthorized with + # `Content-Type: application/zip` and a response body in plaintext, and the + # browser would freak out. + # + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 + def request_format + if request.format == :zip + Mime::Type.lookup_by_extension(:html).ref + else + super + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 79061cd0141..d2e85cabf72 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,18 +1,39 @@ module Gitlab module Diff class File - attr_reader :diff + attr_reader :diff, :diff_refs delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff) + def initialize(diff, diff_refs) @diff = diff + @diff_refs = diff_refs + end + + def old_ref + diff_refs[0] if diff_refs + end + + def new_ref + diff_refs[1] if diff_refs end # Array of Gitlab::DIff::Line objects def diff_lines - @lines ||= parser.parse(raw_diff.lines) + @lines ||= parser.parse(raw_diff.each_line).to_a + end + + def too_large? + diff.too_large? + end + + def highlighted_diff_lines + Gitlab::Diff::Highlight.new(self).highlight + end + + def parallel_diff_lines + Gitlab::Diff::ParallelDiff.new(self).parallelize end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb new file mode 100644 index 00000000000..9429b3ff88d --- /dev/null +++ b/lib/gitlab/diff/highlight.rb @@ -0,0 +1,77 @@ +module Gitlab + module Diff + class Highlight + attr_reader :diff_file, :diff_lines, :raw_lines + + delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + + def initialize(diff_lines) + if diff_lines.is_a?(Gitlab::Diff::File) + @diff_file = diff_lines + @diff_lines = @diff_file.diff_lines + else + @diff_lines = diff_lines + end + @raw_lines = @diff_lines.map(&:text) + end + + def highlight + @diff_lines.map.with_index do |diff_line, i| + diff_line = diff_line.dup + # ignore highlighting for "match" lines + next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + + rich_line = highlight_line(diff_line) || diff_line.text + + if line_inline_diffs = inline_diffs[i] + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + end + + diff_line.text = rich_line + + diff_line + end + end + + private + + def highlight_line(diff_line) + return unless diff_file && diff_file.diff_refs + + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + + case diff_line.type + when 'new', nil + rich_line = new_lines[diff_line.new_pos - 1] + when 'old' + rich_line = old_lines[diff_line.old_pos - 1] + end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + "#{line_prefix}#{rich_line}".html_safe if rich_line + end + + def inline_diffs + @inline_diffs ||= InlineDiff.for_lines(@raw_lines) + end + + def old_lines + return unless diff_file + @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + end + + def new_lines + return unless diff_file + @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) + end + + def processing_args(version) + ref = send("diff_#{version}_ref") + path = send("diff_#{version}_path") + + [ref.project.repository, ref.id, path] + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb new file mode 100644 index 00000000000..789c14518b0 --- /dev/null +++ b/lib/gitlab/diff/inline_diff.rb @@ -0,0 +1,88 @@ +module Gitlab + module Diff + class InlineDiff + attr_accessor :old_line, :new_line, :offset + + def self.for_lines(lines) + local_edit_indexes = self.find_local_edits(lines) + + inline_diffs = [] + + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = lines[old_index] + new_line = lines[new_index] + + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + + inline_diffs[old_index] = old_diffs + inline_diffs[new_index] = new_diffs + end + + inline_diffs + end + + def initialize(old_line, new_line, offset: 0) + @old_line = old_line[offset..-1] + @new_line = new_line[offset..-1] + @offset = offset + end + + def inline_diffs + # Skip inline diff if empty line was replaced with content + return if old_line == "" + + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + + private + + def self.find_local_edits(lines) + line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 + end + + local_edit_indexes + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb new file mode 100644 index 00000000000..dccb717e95d --- /dev/null +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -0,0 +1,115 @@ +module Gitlab + module Diff + class InlineDiffMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(line_inline_diffs) + return rich_line unless line_inline_diffs + + marker_ranges = [] + line_inline_diffs.each do |inline_diff_range| + # Map the inline-diff range based on the raw line to character positions in the rich line + inline_diff_positions = position_mapping[inline_diff_range].flatten + # Turn the array of character positions into ranges + marker_ranges.concat(collapse_ranges(inline_diff_positions)) + end + + offset = 0 + # Mark each range + marker_ranges.each_with_index do |range, i| + class_names = ["idiff"] + class_names << "left" if i == 0 + class_names << "right" if i == marker_ranges.length - 1 + + offset = insert_around_range(rich_line, range, "<span class='#{class_names.join(" ")}'>", "</span>", offset) + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + # Inserts tags around the characters identified by the given range + def insert_around_range(text, range, before, after, offset = 0) + # Just to be sure + return offset if offset + range.end + 1 > text.length + + text.insert(offset + range.begin, before) + offset += before.length + + text.insert(offset + range.end + 1, after) + offset += after.length + + offset + end + end + end +end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0072194606e..03730b435ad 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,7 +1,8 @@ module Gitlab module Diff class Line - attr_reader :type, :text, :index, :old_pos, :new_pos + attr_reader :type, :index, :old_pos, :new_pos + attr_accessor :text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb new file mode 100644 index 00000000000..74f9b3c050a --- /dev/null +++ b/lib/gitlab/diff/parallel_diff.rb @@ -0,0 +1,119 @@ +module Gitlab + module Diff + class ParallelDiff + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + def parallelize + lines = [] + skip_next = false + + highlighted_diff_lines = diff_file.highlighted_diff_lines + highlighted_diff_lines.each do |line| + full_line = line.text + type = line.type + line_code = generate_line_code(diff_file.file_path, line) + line_new = line.new_pos + line_old = line.old_pos + + next_line = diff_file.next_line(line.index) + + if next_line + next_line = highlighted_diff_lines[next_line.index] + next_line_code = generate_line_code(diff_file.file_path, next_line) + next_type = next_line.type + next_line = next_line.text + end + + case type + when 'match', nil + # line in the right panel is the same as in the left one + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + when 'old' + case next_type + when 'new' + # Left side has text removed, right side has text added + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: line_new, + text: next_line, + line_code: next_line_code + } + } + skip_next = true + when 'old', 'nonewline', nil + # Left side has text removed, right side doesn't have any change + # No next line code, no new line number, no new line text + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: nil, + text: "", + line_code: nil + } + } + end + when 'new' + if skip_next + # Change has been already included in previous line so no need to do it again + skip_next = false + next + else + # Change is only on the right side, left side has no change + lines << { + left: { + type: nil, + number: nil, + text: "", + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + end + end + end + lines + end + + private + + def generate_line_code(file_path, line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..d0815fc7eea 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -4,49 +4,56 @@ module Gitlab include Enumerable def parse(lines) + return [] if lines.blank? + @lines = lines - lines_obj = [] line_obj_index = 0 line_old = 1 line_new = 1 type = nil - lines_arr = ::Gitlab::InlineDiff.processing lines - - lines_arr.each do |line| - next if filename?(line) - - full_line = html_escape(line.gsub(/\n/, '')) - full_line = ::Gitlab::InlineDiff.replace_markers full_line - - if line.match(/^@@ -/) - type = "match" - - line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 - line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 #top of file - lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) - line_obj_index += 1 - next - else - type = identification_type(line) - lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) - line_obj_index += 1 - end - - - if line[0] == "+" - line_new += 1 - elsif line[0] == "-" - line_old += 1 - else - line_new += 1 - line_old += 1 + # By returning an Enumerator we make it possible to search for a single line (with #find) + # without having to instantiate all the others that come after it. + Enumerator.new do |yielder| + @lines.each do |line| + next if filename?(line) + + full_line = line.gsub(/\n/, '') + + if line.match(/^@@ -/) + type = "match" + + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 + line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 + + next if line_old <= 1 && line_new <= 1 #top of file + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + next + elsif line[0] == '\\' + type = 'nonewline' + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + else + type = identification_type(line) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + end + + + case line[0] + when "+" + line_new += 1 + when "-" + line_old += 1 + when "\\" + # No increment + else + line_new += 1 + line_old += 1 + end end end - - lines_obj end def empty? @@ -56,24 +63,21 @@ module Gitlab private def filename?(line) - line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', - '--- /tmp/diffy', '+++ /tmp/diffy') + line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit + '--- /tmp/diffy', '+++ /tmp/diffy') end def identification_type(line) - if line[0] == "+" + case line[0] + when "+" "new" - elsif line[0] == "-" + when "-" "old" else nil end end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index a2eb7a70bd2..41f0edcaf7e 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -9,6 +9,7 @@ module Gitlab delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author + delegate :username, to: :author, prefix: :author def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless @@ -49,7 +50,7 @@ module Gitlab end def compare_timeout - compare.timeout if compare + diffs.overflow? if diffs end def reverse_compare? diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2b252b32887..2ca21af5bc8 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -74,7 +74,7 @@ module Gitlab def sent_notification return nil unless reply_key - + SentNotification.for(reply_key) end @@ -82,10 +82,7 @@ module Gitlab attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) attachments.each do |link| - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - reply << "\n\n#{text}" + reply << "\n\n#{link[:markdown]}" end reply diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb new file mode 100644 index 00000000000..2ef50286b1d --- /dev/null +++ b/lib/gitlab/exclusive_lease.rb @@ -0,0 +1,41 @@ +module Gitlab + # This class implements an 'exclusive lease'. We call it a 'lease' + # because it has a set expiry time. We call it 'exclusive' because only + # one caller may obtain a lease for a given key at a time. The + # implementation is intended to work across GitLab processes and across + # servers. It is a 'cheap' alternative to using SQL queries and updates: + # you do not need to change the SQL schema to start using + # ExclusiveLease. + # + # It is important to choose the timeout wisely. If the timeout is very + # high (1 hour) then the throughput of your operation gets very low (at + # most once an hour). If the timeout is lower than how long your + # operation may take then you cannot count on exclusivity. For example, + # if the timeout is 10 seconds and you do an operation which may take 20 + # seconds then two overlapping operations may hold a lease for the same + # key at the same time. + # + class ExclusiveLease + def initialize(key, timeout:) + @key, @timeout = key, timeout + end + + # Try to obtain the lease. Return true on success, + # false if the lease is already taken. + def try_obtain + # Performing a single SET is atomic + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end + + private + + def redis + # Maybe someday we want to use a connection pool... + @redis ||= Redis.new(url: Gitlab::RedisConfig.url) + end + + def redis_key + "gitlab:exclusive_lease:#{@key}" + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 403ebeec474..db580b5e578 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,9 +232,7 @@ module Gitlab return nil if res.nil? - text = "[#{res['alt']}](#{res['url']})" - text = "!#{text}" if res['is_image'] - text + res[:markdown] end def build_attachment_url(rel_url) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index f065cc5e9e9..191bea86ac3 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,8 +1,8 @@ module Gitlab module Git - BLANK_SHA = '0' * 40 - TAG_REF_PREFIX = "refs/tags/" - BRANCH_REF_PREFIX = "refs/heads/" + BLANK_SHA = ('0' * 40).freeze + TAG_REF_PREFIX = "refs/tags/".freeze + BRANCH_REF_PREFIX = "refs/heads/".freeze class << self def ref_name(ref) diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb new file mode 100644 index 00000000000..a088e19d1e7 --- /dev/null +++ b/lib/gitlab/git_post_receive.rb @@ -0,0 +1,60 @@ +module Gitlab + class GitPostReceive + include Gitlab::Identifier + attr_reader :repo_path, :identifier, :changes, :project + + def initialize(repo_path, identifier, changes) + repo_path.gsub!(/\.git\z/, '') + repo_path.gsub!(/\A\//, '') + + @repo_path = repo_path + @identifier = identifier + @changes = deserialize_changes(changes) + + retrieve_project_and_type + end + + def wiki? + @type == :wiki + end + + def regular_project? + @type == :project + end + + def identify(revision) + super(identifier, project, revision) + end + + private + + def retrieve_project_and_type + @type = :project + @project = Project.find_with_namespace(@repo_path) + + if @repo_path.end_with?('.wiki') && !@project + @type = :wiki + @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, '')) + end + end + + def deserialize_changes(changes) + changes = Base64.decode64(changes) unless changes.include?(' ') + changes = utf8_encode_changes(changes) + changes.lines + end + + def utf8_encode_changes(changes) + changes = changes.dup + + changes.force_encoding('UTF-8') + return changes if changes.valid_encoding? + + # Convert non-UTF-8 branch/tag names to UTF-8 so they can be dumped as JSON. + detection = CharlockHolmes::EncodingDetector.detect(changes) + return changes unless detection && detection[:encoding] + + CharlockHolmes::Converter.convert(changes, detection[:encoding], 'UTF-8') + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b5720f6e2cb..172c5441e36 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -12,39 +14,86 @@ module Gitlab end def execute - #Issues && Comments + import_issues && import_pull_requests && import_wiki + end + + private + + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? - - body = @formatter.author_line(issue.user.login) - body += issue.body || "" + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - if issue.comments > 0 - body += @formatter.comments_header + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end + end + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| + pull_request = PullRequestFormatter.new(project, raw_data) + + if pull_request.valid? + merge_request = MergeRequest.new(pull_request.attributes) - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) - ) + if merge_request.save + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) + end end end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end - private + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) + end + + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) + end + end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + + true + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + raise Projects::ImportService::Error, e.message + else + true + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..4e507b090e8 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,105 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def number + raw_data.number + end + + def valid? + !cross_project? && source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def cross_project? + source_repo.present? && target_repo.present? && source_repo.id != target_repo.id + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_repo + raw_data.head.repo + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_repo + raw_data.base.repo + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6c592ff469c --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(/\.git\z/, ".wiki.git") + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..850b73244c6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source) #Issues && Comments issues = client.issues(project_identifier) diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb new file mode 100644 index 00000000000..cac76442321 --- /dev/null +++ b/lib/gitlab/highlight.rb @@ -0,0 +1,39 @@ +module Gitlab + class Highlight + def self.highlight(blob_name, blob_content, nowrap: true) + new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) + end + + def self.highlight_lines(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + blob.load_all_data!(repository) + highlight(file_name, blob.data).lines.map!(&:html_safe) + end + + def initialize(blob_name, blob_content, nowrap: true) + @formatter = rouge_formatter(nowrap: nowrap) + @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + end + + def highlight(text, continue: true) + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + rescue + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + private + + def rouge_formatter(options = {}) + options = options.reverse_merge( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + + Rouge::Formatters::HTMLGitlab.new(options) + end + end +end diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb deleted file mode 100644 index 44507bde25d..00000000000 --- a/lib/gitlab/inline_diff.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Gitlab - class InlineDiff - class << self - - START = "#!idiff-start!#" - FINISH = "#!idiff-finish!#" - - def processing(diff_arr) - indexes = _indexes_of_changed_lines diff_arr - - indexes.each do |index| - first_line = diff_arr[index+1] - second_line = diff_arr[index+2] - - # Skip inline diff if empty line was replaced with content - next if first_line == "-\n" - - first_token = find_first_token(first_line, second_line) - apply_first_token(diff_arr, index, first_token) - - last_token = find_last_token(first_line, second_line, first_token) - apply_last_token(diff_arr, index, last_token) - end - - diff_arr - end - - def apply_first_token(diff_arr, index, first_token) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end - end - - def apply_last_token(diff_arr, index, last_token) - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - end - - def find_first_token(first_line, second_line) - max_length = [first_line.size, second_line.size].max - first_the_same_symbols = 0 - - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_line[0..first_the_same_symbols][1..-1] - end - - def find_last_token(first_line, second_line, first_token) - max_length = [first_line.size, second_line.size].max - last_the_same_symbols = 0 - - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - - if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - - last_the_same_symbols += 1 - first_line[last_the_same_symbols..-1] - end - - def _indexes_of_changed_lines(diff_arr) - chain_of_first_symbols = "" - diff_arr.each_with_index do |line, i| - chain_of_first_symbols += line[0] - end - chain_of_first_symbols.gsub!(/[^\-\+]/, "#") - - offset = 0 - indexes = [] - while index = chain_of_first_symbols.index("#-+#", offset) - indexes << index - offset = index + 1 - end - indexes - end - - def replace_markers(line) - line.gsub!(START, "<span class='idiff'>") - line.gsub!(FINISH, "</span>") - line - end - end - end -end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..da4435c7308 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,20 +32,20 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block + user.ldap_block false else - user.activate if user.blocked? && !ldap_config.block_auto_created_users + user.activate if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.block + user.ldap_block false end rescue @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index aef08c97d1d..b84c81f1a6c 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -24,34 +24,41 @@ module Gitlab update_user_attributes end + def save + super('LDAP') + end + # instance methods def gl_user @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider - self.class.find_by_uid_and_provider( - auth_hash.uid, auth_hash.provider) + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? end def update_user_attributes - return unless persisted? + if persisted? + if auth_hash.has_email? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) + # For a new identity set extern_uid to the LDAP DN + # For an existing identity with matching email but changed DN, update the DN. + # For an existing identity with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid + end - # For a new user set extern_uid to the LDAP DN - # For an existing user with matching email but changed DN, update the DN. - # For an existing user with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid + gl_user.ldap_email = auth_hash.has_email? gl_user end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 8f3f43c0e91..699d8b9fc07 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab module Markdown class Pipeline diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 2d266ccfe9e..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -6,16 +6,20 @@ module Gitlab METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - def self.pool_size - current_application_settings[:metrics_pool_size] || 16 - end - - def self.timeout - current_application_settings[:metrics_timeout] || 10 + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 + } end def self.enabled? - current_application_settings[:metrics_enabled] || false + settings[:enabled] || false end def self.mri? @@ -26,32 +30,13 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= - (current_application_settings[:metrics_method_call_threshold] || 10) + @method_call_threshold ||= settings[:method_call_threshold] end def self.pool @pool end - def self.hostname - @hostname - end - - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) @@ -85,19 +70,15 @@ module Gitlab value.to_s.gsub('=', '\\=') end - @hostname = Socket.gethostname - # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? - @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = current_application_settings[:metrics_host] - user = current_application_settings[:metrics_username] - pw = current_application_settings[:metrics_password] - port = current_application_settings[:metrics_port] + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] InfluxDB::Client. - new(udp: { host: host, port: port }, username: user, password: pw) + new(udp: { host: host, port: port }) end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 06fc2f25948..face1921d2e 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -106,23 +106,41 @@ module Gitlab if type == :instance target = mod label = "#{mod.name}##{name}" + method = mod.instance_method(name) else target = mod.singleton_class label = "#{mod.name}.#{name}" + method = mod.method(name) end + # Some code out there (e.g. the "state_machine" Gem) checks the arity of + # a method to make sure it only passes arguments when the method expects + # any. If we were to always overwrite a method to take an `*args` + # signature this would break things. As a result we'll make sure the + # generated method _only_ accepts regular arguments if the underlying + # method also accepts them. + if method.arity == 0 + args_signature = '&block' + else + args_signature = '*args, &block' + end + + send_signature = "__send__(#{alias_name.inspect}, #{args_signature})" + target.class_eval <<-EOF, __FILE__, __LINE__ + 1 alias_method #{alias_name.inspect}, #{name.inspect} - def #{name}(*args, &block) + def #{name}(#{args_signature}) trans = Gitlab::Metrics::Instrumentation.transaction if trans start = Time.now - retval = __send__(#{alias_name.inspect}, *args, &block) + retval = #{send_signature} duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) @@ -130,7 +148,7 @@ module Gitlab retval else - __send__(#{alias_name.inspect}, *args, &block) + #{send_signature} end end EOF diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 753008df99a..7ea9555cc8c 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -17,11 +17,8 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash { - series: @series, - tags: @tags.merge( - hostname: Metrics.hostname, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' - ), + series: @series, + tags: @tags, values: @values, timestamp: @created_at.to_i * 1_000_000_000 } diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb deleted file mode 100644 index fe97d7a0534..00000000000 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Metrics - # Class for producing SQL queries with sensitive data stripped out. - class ObfuscatedSQL - REPLACEMENT = / - \d+(\.\d+)? # integers, floats - | '.+?' # single quoted strings - | \/.+?(?<!\\)\/ # regexps (including escaped slashes) - /x - - MYSQL_REPLACEMENTS = / - ".+?" # double quoted strings - /x - - # Regex to replace consecutive placeholders with a single one indicating - # the length. This can be useful when a "IN" statement uses thousands of - # IDs (storing this would just be a waste of space). - CONSECUTIVE = /(\?(\s*,\s*)?){2,}/ - - # sql - The raw SQL query as a String. - def initialize(sql) - @sql = sql - end - - # Returns a new, obfuscated SQL query. - def to_s - regex = REPLACEMENT - - if Gitlab::Database.mysql? - regex = Regexp.union(regex, MYSQL_REPLACEMENTS) - end - - sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match| - "#{match.count(',') + 1} values" - end - - # InfluxDB escapes double quotes upon output, so lets get rid of them - # whenever we can. - if Gitlab::Database.postgresql? - sql = sql.delete('"') - end - - sql.tr("\n", ' ') - end - end - end -end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -32,17 +32,15 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.add_tag(:request_method, env['REQUEST_METHOD']) - trans.add_tag(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_method, env['REQUEST_METHOD']) trans end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 998578e1c0a..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,9 +7,14 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) - @interval = interval - @metrics = [] + def initialize(interval = Metrics.settings[:sample_interval]) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -50,12 +55,11 @@ module Gitlab end def sample_memory_usage - @metrics << Metric.new('memory_usage', value: System.memory_usage) + add_metric('memory_usage', value: System.memory_usage) end def sample_file_descriptors - @metrics << Metric. - new('file_descriptors', value: System.file_descriptor_count) + add_metric('file_descriptors', value: System.file_descriptor_count) end if Metrics.mri? @@ -69,7 +73,7 @@ module Gitlab counts['Symbol'] = Symbol.all_symbols.length counts.each do |name, count| - @metrics << Metric.new('object_counts', { count: count }, type: name) + add_metric('object_counts', { count: count }, type: name) end end else @@ -91,7 +95,34 @@ module Gitlab stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - @metrics << Metric.new('gc_statistics', stats) + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7e0dcf99d92..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,6 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end @@ -32,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d947c128ce2..8008b3bc895 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -1,44 +1,18 @@ module Gitlab module Metrics module Subscribers - # Class for tracking raw SQL queries. - # - # Queries are obfuscated before being logged to ensure no private data is - # exposed via InfluxDB/Grafana. + # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber attach_to :active_record - SERIES = 'sql_queries' - def sql(event) return unless current_transaction - values = values_for(event) - tags = tags_for(event) - - current_transaction.add_metric(SERIES, values, tags) + current_transaction.increment(:sql_duration, event.duration) end private - def values_for(event) - { duration: event.duration } - end - - def tags_for(event) - sql = ObfuscatedSQL.new(event.payload[:sql]).to_s - tags = { sql: sql } - - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags - end - def current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index a61dbd989e7..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,45 +4,64 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - SERIES = 'transactions' + attr_reader :tags, :values - attr_reader :uuid, :tags + attr_accessor :action def self.current Thread.current[THREAD_KEY] end - # name - The name of this transaction as a String. - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] - @uuid = SecureRandom.uuid @started_at = nil @finished_at = nil - @tags = {} + @values = Hash.new(0) + @tags = {} + @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def increment(name, value) + @values[name] += value + end - @metrics << Metric.new(series, values, tags) + def set(name, value) + @values[name] = value end def add_tag(key, value) @@ -55,11 +74,29 @@ module Gitlab end def track_self - add_metric(SERIES, { duration: duration }, @tags) + values = { duration: duration, allocated_memory: allocated_memory } + + @values.each do |name, value| + values[name] = value + end + + add_metric('transactions', values, @tags) end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) + end + + def sidekiq? + Sidekiq.server? end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb new file mode 100644 index 00000000000..50b0dd32380 --- /dev/null +++ b/lib/gitlab/middleware/go.rb @@ -0,0 +1,50 @@ +# A dumb middleware that returns a Go HTML document if the go-get=1 query string +# is used irrespective if the namespace/project exists +module Gitlab + module Middleware + class Go + def initialize(app) + @app = app + end + + def call(env) + request = Rack::Request.new(env) + + if go_request?(request) + render_go_doc(request) + else + @app.call(env) + end + end + + private + + def render_go_doc(request) + body = go_body(request) + response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' }) + response.finish + end + + def go_request?(request) + request["go-get"].to_i == 1 && request.env["PATH_INFO"].present? + end + + def go_body(request) + base_url = Gitlab.config.gitlab.url + # Go subpackages may be in the form of namespace/project/path1/path2/../pathN + # We can just ignore the paths and leave the namespace/project + path_info = request.env["PATH_INFO"] + path_info.sub!(/^\//, '') + project_path = path_info.split('/').first(2).join('/') + request_url = URI.join(base_url, project_path) + domain_path = strip_url(request_url.to_s) + + "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n"; + end + + def strip_url(url) + url.gsub(/\Ahttps?:\/\//, '') + end + end + end +end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index ea6b0ee796d..71cf6a0d886 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -53,13 +53,10 @@ module Gitlab object_kind: "note", user: user.hook_attrs, project_id: project.id, - repository: { - name: project.name, - url: project.url_to_repo, - description: project.description, - homepage: project.web_url, - }, - object_attributes: note.hook_attrs + project: project.hook_attrs, + object_attributes: note.hook_attrs, + # DEPRECATED + repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } base_data[:object_attributes][:url] = diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index ba31599432b..36e5c2670bb 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,6 +32,10 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end + def has_email? + get_info(:email).present? + end + private def info @@ -46,8 +50,8 @@ module Gitlab def username_and_email @username_and_email ||= begin - username = get_info(:username) || get_info(:nickname) - email = get_info(:email) + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence username ||= generate_username(email) if email email ||= generate_temporarily_email(username) if username diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f1a362f5303..832fb08a526 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -26,7 +26,7 @@ module Gitlab gl_user.try(:valid?) end - def save + def save(provider = 'OAuth') unauthorized_to_create unless gl_user if needs_blocking? @@ -36,10 +36,10 @@ module Gitlab gl_user.save! end - log.info "(OAuth) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e - log.info "(OAuth) Error saving user: #{gl_user.errors.full_messages}" + log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}" return self, e.record.errors end @@ -105,13 +105,18 @@ module Gitlab end def signup_enabled? - Gitlab.config.omniauth.allow_single_sign_on + providers = Gitlab.config.omniauth.allow_single_sign_on + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users - else + else Gitlab.config.omniauth.block_auto_created_users end end @@ -135,15 +140,18 @@ module Gitlab def user_attributes # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? - username = ldap_person.username - email = ldap_person.email.first - else - username = auth_hash.username - email = auth_hash.email + username = ldap_person.username.presence + email = ldap_person.email.first.presence end - + + username ||= auth_hash.username + email ||= auth_hash.email + + name = auth_hash.name + name = ::Namespace.clean_path(username) if name.strip.empty? + { - name: auth_hash.name, + name: name, username: ::Namespace.clean_path(username), email: email, password: auth_hash.password, diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb new file mode 100644 index 00000000000..746ec283330 --- /dev/null +++ b/lib/gitlab/other_markup.rb @@ -0,0 +1,24 @@ +module Gitlab + # Parser/renderer for markups without other special support code. + module OtherMarkup + + # Public: Converts the provided markup into HTML. + # + # input - the source text in a markup format + # context - a Hash with the template context: + # :commit + # :project + # :project_wiki + # :requested_path + # :ref + # + def self.render(file_name, input, context) + html = GitHub::Markup.render(file_name, input). + force_encoding(input.encoding) + + html = Banzai.post_process(html, context) + + html.html_safe + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 70de6a74e76..71c5b6801fb 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,8 +2,9 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(project_id, query, repository_ref = nil) - @project = Project.find(project_id) + def initialize(current_user, project, query, repository_ref = nil) + @current_user = current_user + @project = project @repository_ref = if repository_ref.present? repository_ref else @@ -73,7 +74,7 @@ module Gitlab end def notes - Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') + project.notes.user.search(query).order('updated_at DESC') end def commits @@ -84,8 +85,8 @@ module Gitlab end end - def limit_project_ids - [project.id] + def project_ids_relation + project end end end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index 4f9cdef3869..97d1edab9c1 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -22,6 +22,8 @@ module Gitlab # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + commits = Array(commits) + # Total commits count commits_count = commits.size @@ -47,25 +49,21 @@ module Gitlab user_id: user.id, user_name: user.name, user_email: user.email, + user_avatar: user.avatar_url, project_id: project.id, - repository: { - name: project.name, - url: project.url_to_repo, - description: project.description, - homepage: project.web_url, - git_http_url: project.http_url_to_repo, - git_ssh_url: project.ssh_url_to_repo, - visibility_level: project.visibility_level - }, + project: project.hook_attrs, commits: commit_attrs, - total_commits_count: commits_count + total_commits_count: commits_count, + # DEPRECATED + repository: project.hook_attrs.slice(:name, :url, :description, :homepage, + :git_http_url, :git_ssh_url, :visibility_level) } data end # This method provide a sample data generated with - # existing project and commits to test web hooks + # existing project and commits to test webhooks def build_sample(project, user) commits = project.repository.commits(project.default_branch, nil, 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb new file mode 100644 index 00000000000..4949c6db539 --- /dev/null +++ b/lib/gitlab/redis_config.rb @@ -0,0 +1,30 @@ +module Gitlab + class RedisConfig + attr_reader :url + + def self.url + new.url + end + + def self.redis_store_options + url = new.url + redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index be795649e59..4d830aa45e1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor @@ -19,7 +17,7 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label merge_request snippet commit commit_range).each do |type| + %i(user label milestone merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do @references[type] ||= references(type, reference_context) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 53ab2686b43..ace906a6f59 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -34,29 +34,29 @@ module Gitlab def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git)\z/.freeze + @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end def project_path_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.git'" \ + "Cannot start with '-', end in '.git' or end in '.atom'" \ end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.]*\z/.freeze + @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'. " end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze end def file_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb new file mode 100644 index 00000000000..b1e30110ef5 --- /dev/null +++ b/lib/gitlab/saml/user.rb @@ -0,0 +1,47 @@ +# SAML extension for User model +# +# * Find GitLab user based on SAML uid and provider +# * Create new user from SAML data +# +module Gitlab + module Saml + class User < Gitlab::OAuth::User + + def save + super('SAML') + end + + def gl_user + @user ||= find_by_uid_and_provider + + if auto_link_ldap_user? + @user ||= find_or_create_ldap_user + end + + if auto_link_saml_enabled? + @user ||= find_by_email + end + + if signup_enabled? + @user ||= build_new_user + end + + @user + end + + def find_by_email + if auth_hash.has_email? + user = ::User.find_by(email: auth_hash.email.downcase) + user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user + user + end + end + + protected + + def auto_link_saml_enabled? + Gitlab.config.omniauth.auto_link_saml_user + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 2ab2d4af797..f8ab2b1f09e 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,13 +1,14 @@ module Gitlab class SearchResults - attr_reader :query + attr_reader :current_user, :query - # Limit search results by passed project ids + # Limit search results by passed projects # It allows us to search only for projects user has access to - attr_reader :limit_project_ids + attr_reader :limit_projects - def initialize(limit_project_ids, query) - @limit_project_ids = limit_project_ids || Project.all + def initialize(current_user, limit_projects, query) + @current_user = current_user + @limit_projects = limit_projects || Project.all @query = Shellwords.shellescape(query) if query.present? end @@ -27,7 +28,8 @@ module Gitlab end def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count + @total_count ||= projects_count + issues_count + merge_requests_count + + milestones_count end def projects_count @@ -53,27 +55,29 @@ module Gitlab private def projects - Project.where(id: limit_project_ids).search(query) + limit_projects.search(query) end def issues - issues = Issue.where(project_id: limit_project_ids) + issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) + if query =~ /#(\d+)\z/ issues = issues.where(iid: $1) else issues = issues.full_search(query) end + issues.order('updated_at DESC') end def milestones - milestones = Milestone.where(project_id: limit_project_ids) + milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) milestones.order('updated_at DESC') end def merge_requests - merge_requests = MergeRequest.in_projects(limit_project_ids) + merge_requests = MergeRequest.in_projects(project_ids_relation) if query =~ /[#!](\d+)\z/ merge_requests = merge_requests.where(iid: $1) else @@ -89,5 +93,9 @@ module Gitlab def per_page 20 end + + def project_ids_relation + limit_projects.select(:id).reorder(nil) + end end end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 938219efdb2..e0e74ff8359 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -1,18 +1,20 @@ module Gitlab class SnippetSearchResults < SearchResults - attr_reader :limit_snippet_ids + include SnippetsHelper - def initialize(limit_snippet_ids, query) - @limit_snippet_ids = limit_snippet_ids + attr_reader :limit_snippets + + def initialize(limit_snippets, query) + @limit_snippets = limit_snippets @query = query end def objects(scope, page = nil) case scope when 'snippet_titles' - Kaminari.paginate_array(snippet_titles).page(page).per(per_page) + snippet_titles.page(page).per(per_page) when 'snippet_blobs' - Kaminari.paginate_array(snippet_blobs).page(page).per(per_page) + snippet_blobs.page(page).per(per_page) else super end @@ -33,99 +35,15 @@ module Gitlab private def snippet_titles - Snippet.where(id: limit_snippet_ids).search(query).order('updated_at DESC') + limit_snippets.search(query).order('updated_at DESC') end def snippet_blobs - search = Snippet.where(id: limit_snippet_ids).search_code(query) - search = search.order('updated_at DESC').to_a - snippets = [] - search.each { |e| snippets << chunk_snippet(e) } - snippets + limit_snippets.search_code(query).order('updated_at DESC') end def default_scope 'snippet_blobs' end - - # Get an array of line numbers surrounding a matching - # line, bounded by min/max. - # - # @returns Array of line numbers - def bounded_line_numbers(line, min, max) - lower = line - surrounding_lines > min ? line - surrounding_lines : min - upper = line + surrounding_lines < max ? line + surrounding_lines : max - (lower..upper).to_a - end - - # Returns a sorted set of lines to be included in a snippet preview. - # This ensures matching adjacent lines do not display duplicated - # surrounding code. - # - # @returns Array, unique and sorted. - def matching_lines(lined_content) - used_lines = [] - lined_content.each_with_index do |line, line_number| - used_lines.concat bounded_line_numbers( - line_number, - 0, - lined_content.size - ) if line.include?(query) - end - - used_lines.uniq.sort - end - - # 'Chunkify' entire snippet. Splits the snippet data into matching lines + - # surrounding_lines() worth of unmatching lines. - # - # @returns a hash with {snippet_object, snippet_chunks:{data,start_line}} - def chunk_snippet(snippet) - lined_content = snippet.content.split("\n") - used_lines = matching_lines(lined_content) - - snippet_chunk = [] - snippet_chunks = [] - snippet_start_line = 0 - last_line = -1 - - # Go through each used line, and add consecutive lines as a single chunk - # to the snippet chunk array. - used_lines.each do |line_number| - if last_line < 0 - # Start a new chunk. - snippet_start_line = line_number - snippet_chunk << lined_content[line_number] - elsif last_line == line_number - 1 - # Consecutive line, continue chunk. - snippet_chunk << lined_content[line_number] - else - # Non-consecutive line, add chunk to chunk array. - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Start a new chunk. - snippet_chunk = [lined_content[line_number]] - snippet_start_line = line_number - end - last_line = line_number - end - # Add final chunk to chunk array - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Return snippet with chunk array - { snippet_object: snippet, snippet_chunks: snippet_chunks } - end - - # Defines how many unmatching lines should be - # included around the matching lines in a snippet - def surrounding_lines - 3 - end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 4885baf9526..d1b42c1f9b9 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -3,7 +3,7 @@ module Gitlab def self.allowed?(user) return false if user.blocked? - if user.requires_ldap_check? + if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb new file mode 100644 index 00000000000..c3ddd4c2680 --- /dev/null +++ b/lib/gitlab/workhorse.rb @@ -0,0 +1,40 @@ +require 'base64' +require 'json' + +module Gitlab + class Workhorse + SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' + + class << self + def send_git_blob(repository, blob) + params = { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id, + } + + [ + SEND_DATA_HEADER, + "git-blob:#{encode(params)}", + ] + end + + def send_git_archive(project, ref, format) + format ||= 'tar.gz' + format.downcase! + params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + raise "Repository or ref not found" if params.empty? + + [ + SEND_DATA_HEADER, + "git-archive:#{encode(params)}", + ] + end + + protected + + def encode(hash) + Base64.urlsafe_encode64(JSON.dump(hash)) + end + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index c5f07c8b508..d95e7023d2e 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -38,6 +38,7 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" @@ -48,7 +49,7 @@ test -f /etc/default/gitlab && . /etc/default/gitlab # Switch to the app_user if it is not he/she who is running the script. if [ `whoami` != "$app_user" ]; then - eval su - "$app_user" -s $shell_path -c $(echo \")$0 "$@"$(echo \"); exit; + eval su - "$app_user" -c $(echo \")$shell_path -l -c \'$0 "$@"\'$(echo \"); exit; fi # Switch to the gitlab path, exit on failure. @@ -218,7 +219,7 @@ start_gitlab() { echo "The Unicorn web server already running with pid $wpid, not restarting." else # Remove old socket if it exists - rm -f "$socket_path"/gitlab.socket 2>/dev/null + rm -f "$rails_socket" 2>/dev/null # Start the web server RAILS_ENV=$RAILS_ENV bin/web start fi @@ -233,10 +234,12 @@ start_gitlab() { if [ "$gitlab_workhorse_status" = "0" ]; then echo "The gitlab-workhorse is already running with pid $spid, not restarting" else - # No need to remove a socket, gitlab-workhorse does this itself + # No need to remove a socket, gitlab-workhorse does this itself. + # Because gitlab-workhorse has multiple executables we need to fix + # the PATH. $app_root/bin/daemon_with_pidfile $gitlab_workhorse_pid_path \ - $app_root/../gitlab-workhorse/gitlab-workhorse \ - $gitlab_workhorse_options \ + /usr/bin/env PATH=$gitlab_workhorse_dir:$PATH \ + gitlab-workhorse $gitlab_workhorse_options \ >> $gitlab_workhorse_log 2>&1 & fi diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 1937ca582b0..cc8617b72ca 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,12 +30,20 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# The directory where the gitlab-workhorse binaries are. Usually +# /home/git/gitlab-workhorse . +gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" + # The -listenXxx settings determine where gitlab-workhorse -# listens for connections from NGINX. To listen on localhost:8181, write -# '-listenNetwork tcp -listenAddr localhost:8181'. -# The -authBackend setting tells gitlab-workhorse where it can reach -# Unicorn. +# listens for connections from the web server. By default it listens to a +# socket. To listen on TCP connections (needed by Apache) change to: +# '-listenNetwork tcp -listenAddr 127.0.0.1:8181' +# +# The -authBackend setting tells gitlab-workhorse where it can reach Unicorn. +# For relative URL support change to: +# '-authBackend http://127.0.0.1/8080/gitlab' +# Read more in http://doc.gitlab.com/ce/install/relative_url.html gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index fc5475c4eef..1324e4cd267 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -30,7 +30,6 @@ server { listen [::]:80 default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - root /home/git/gitlab/public; ## See app/controllers/application_controller.rb for headers set @@ -57,4 +56,14 @@ server { proxy_pass http://gitlab-workhorse; } + + error_page 404 /404.html; + error_page 422 /422.html; + error_page 500 /500.html; + error_page 502 /502.html; + location ~ ^/(404|422|500|502)\.html$ { + root /home/git/gitlab/public; + internal; + } + } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 1e5f85413ec..af6ea9ed706 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -45,7 +45,6 @@ server { listen [::]:443 ipv6only=on ssl default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice - root /home/git/gitlab/public; ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ @@ -101,4 +100,13 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://gitlab-workhorse; } + + error_page 404 /404.html; + error_page 422 /422.html; + error_page 500 /500.html; + error_page 502 /502.html; + location ~ ^/(404|422|500|502)\.html$ { + root /home/git/gitlab/public; + internal; + } } diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 5d4e0740373..d5a402907d8 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%W(brakeman --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%W(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 1728dda72cf..51e746ef923 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,11 +1,21 @@ namespace :cache do + CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + desc "GitLab | Clear redis cache" task :clear => :environment do - # Hack into Rails.cache until https://github.com/redis-store/redis-store/pull/225 - # is accepted (I hope) and we can update the redis-store gem. - redis_store = Rails.cache.instance_variable_get(:@data) - redis_store.keys.each_slice(1000) do |key_slice| - redis_store.del(*key_slice) + redis = Redis.new(url: Gitlab::RedisConfig.url) + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", + count: CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake new file mode 100644 index 00000000000..cfaf4a129b1 --- /dev/null +++ b/lib/tasks/gemojione.rake @@ -0,0 +1,122 @@ +# This task will generate a standard and Retina sprite of all of the current +# Gemojione Emojis, with the accompanying SCSS map. +# +# It will not appear in `rake -T` output, and the dependent gems are not +# included in the Gemfile by default, because this task will only be needed +# occasionally, such as when new Emojis are added to Gemojione. + +begin + require 'sprite_factory' + require 'rmagick' +rescue LoadError + # noop +end + +namespace :gemojione do + task sprite: :environment do + check_requirements! + + SIZE = 20 + RETINA = SIZE * 2 + + Dir.mktmpdir do |tmpdir| + # Copy the Gemojione assets to the temporary folder for resizing + FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + + Dir.chdir(tmpdir) do + Dir["**/*.png"].each do |png| + resize!(File.join(tmpdir, png), SIZE) + end + end + + style_path = Rails.root.join(*%w(app assets stylesheets pages emojis.scss)) + + # Combine the resized assets into a packed sprite and re-generate the SCSS + SpriteFactory.cssurl = "image-url('$IMAGE')" + SpriteFactory.run!(File.join(tmpdir, 'images'), { + output_style: style_path, + output_image: "app/assets/images/emoji.png", + selector: '.emoji-', + style: :scss, + nocomments: true, + pngcrush: true, + layout: :packed + }) + + # SpriteFactory's SCSS is a bit too verbose for our purposes here, so + # let's simplify it + system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path})) + system(%Q(sed -i '' "s/ no-repeat//" #{style_path})) + system(%Q(sed -i '' "s/ 0px/ 0/" #{style_path})) + + # Append a generic rule that applies to all Emojis + File.open(style_path, 'a') do |f| + f.puts + f.puts <<-CSS.strip_heredoc + .emoji-icon { + background-image: image-url('emoji.png'); + background-repeat: no-repeat; + height: #{SIZE}px; + width: #{SIZE}px; + + @media only screen and (-webkit-min-device-pixel-ratio: 2), + only screen and (min--moz-device-pixel-ratio: 2), + only screen and (-o-min-device-pixel-ratio: 2/1), + only screen and (min-device-pixel-ratio: 2), + only screen and (min-resolution: 192dpi), + only screen and (min-resolution: 2dppx) { + background-image: image-url('emoji@2x.png'); + background-size: 840px 820px; + } + } + CSS + end + end + + # Now do it again but for Retina + Dir.mktmpdir do |tmpdir| + # Copy the Gemojione assets to the temporary folder for resizing + FileUtils.cp_r(Gemojione.index.images_path, tmpdir) + + Dir.chdir(tmpdir) do + Dir["**/*.png"].each do |png| + resize!(File.join(tmpdir, png), RETINA) + end + end + + # Combine the resized assets into a packed sprite and re-generate the SCSS + SpriteFactory.run!(File.join(tmpdir, 'images'), { + output_image: "app/assets/images/emoji@2x.png", + style: false, + nocomments: true, + pngcrush: true, + layout: :packed + }) + end + end + + def check_requirements! + return if defined?(SpriteFactory) && defined?(Magick) + + puts <<-MSG.strip_heredoc + This task is disabled by default and should only be run when the Gemojione + gem is updated with new Emojis. + + To enable this task, *temporarily* add the following lines to Gemfile and + re-bundle: + + gem 'sprite-factory' + gem 'rmagick' + MSG + + exit 1 + end + + def resize!(image_path, size) + # Resize the image in-place, save it, and free the object + image = Magick::Image.read(image_path).first + image.resize!(size, size) + image.write(image_path) { self.quality = 100 } + image.destroy! + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 0469c5a61c3..27ed57efe55 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -16,7 +16,6 @@ namespace :gitlab do check_git_config check_database_config_exists - check_database_is_not_sqlite check_migrations_are_up check_orphaned_group_members check_gitlab_config_exists @@ -90,24 +89,6 @@ namespace :gitlab do end end - def check_database_is_not_sqlite - print "Database is SQLite ... " - - database_config_file = Rails.root.join("config", "database.yml") - - unless File.read(database_config_file) =~ /adapter:\s+sqlite/ - puts "no".green - else - puts "yes".red - puts "Please fix this by removing the SQLite entry from the database.yml".blue - for_more_information( - "https://github.com/gitlabhq/gitlabhq/wiki/Migrate-from-SQLite-to-MySQL", - see_database_guide - ) - fix_and_rerun - end - end - def check_gitlab_config_exists print "GitLab config exists? ... " @@ -285,7 +266,7 @@ namespace :gitlab do unless File.directory?(Rails.root.join('public/uploads')) puts "no".red try_fixing_it( - "sudo -u #{gitlab_user} mkdir -m 750 #{Rails.root}/public/uploads" + "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" ) for_more_information( see_installation_guide_section "GitLab" @@ -297,21 +278,22 @@ namespace :gitlab do upload_path = File.realpath(Rails.root.join('public/uploads')) upload_path_tmp = File.join(upload_path, 'tmp') - if File.stat(upload_path).mode == 040750 + if File.stat(upload_path).mode == 040700 unless Dir.exists?(upload_path_tmp) puts 'skipped (no tmp uploads folder yet)'.magenta return end - # if tmp upload dir has incorrect permissions, assume others do as well - if File.stat(upload_path_tmp).mode == 040755 && File.owned?(upload_path_tmp) # verify drwxr-xr-x permissions + # If tmp upload dir has incorrect permissions, assume others do as well + # Verify drwx------ permissions + if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) puts "yes".green else puts "no".red try_fixing_it( "sudo chown -R #{gitlab_user} #{upload_path}", "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0755 {} \\;" + "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" ) for_more_information( see_installation_guide_section "GitLab" @@ -321,7 +303,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "sudo chmod 0750 #{upload_path}", + "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" ) for_more_information( see_installation_guide_section "GitLab" @@ -431,7 +413,7 @@ namespace :gitlab do try_fixing_it( "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", "sudo chmod -R ug-s #{repo_base_path}", - "find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" ) for_more_information( see_installation_guide_section "GitLab Shell" @@ -746,13 +728,15 @@ namespace :gitlab do def check_imap_authentication print "IMAP server credentials are correct? ... " - config = Gitlab.config.incoming_email + config_path = Rails.root.join('config', 'mail_room.yml') + config_file = YAML.load(ERB.new(File.read(config_path)).result) + config = config_file[:mailboxes].first if config begin - imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl) - imap.starttls if config.start_tls - imap.login(config.user, config.password) + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) connected = true rescue connected = false @@ -929,7 +913,7 @@ namespace :gitlab do end def check_git_version - required_version = Gitlab::VersionInfo.new(1, 7, 10) + required_version = Gitlab::VersionInfo.new(2, 7, 3) current_version = Gitlab::VersionInfo.parse(run(%W(#{Gitlab.config.git.bin_path} --version))) puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index ebe516ec879..d33b5b31e18 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -2,6 +2,11 @@ module Gitlab class TaskAbortedByUserError < StandardError; end end +String.disable_colorization = true unless STDOUT.isatty + +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + namespace :gitlab do # Ask if the user wants to continue diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 76e443e55ee..cc0f668474e 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -1,13 +1,13 @@ namespace :gitlab do namespace :web_hook do - desc "GitLab | Adds a web hook to the projects" + desc "GitLab | Adds a webhook to the projects" task :add => :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] projects = find_projects(namespace_path) - puts "Adding web hook '#{web_hook_url}' to:" + puts "Adding webhook '#{web_hook_url}' to:" projects.find_each(batch_size: 1000) do |project| print "- #{project.name} ... " web_hook = project.hooks.new(url: web_hook_url) @@ -20,7 +20,7 @@ namespace :gitlab do end end - desc "GitLab | Remove a web hook from the projects" + desc "GitLab | Remove a webhook from the projects" task :rm => :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] @@ -28,12 +28,12 @@ namespace :gitlab do projects = find_projects(namespace_path) projects_ids = projects.pluck(:id) - puts "Removing web hooks with the url '#{web_hook_url}' ... " + puts "Removing webhooks with the url '#{web_hook_url}' ... " count = WebHook.where(url: web_hook_url, project_id: projects_ids, type: 'ProjectHook').delete_all - puts "#{count} web hooks were removed." + puts "#{count} webhooks were removed." end - desc "GitLab | List web hooks" + desc "GitLab | List webhooks" task :list => :environment do namespace_path = ENV['NAMESPACE'] @@ -43,7 +43,7 @@ namespace :gitlab do puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}" end - puts "\n#{web_hooks.size} web hooks found." + puts "\n#{web_hooks.size} webhooks found." end end diff --git a/lib/tasks/scss-lint.rake b/lib/tasks/scss-lint.rake new file mode 100644 index 00000000000..250fd8699e4 --- /dev/null +++ b/lib/tasks/scss-lint.rake @@ -0,0 +1,10 @@ +unless Rails.env.production? + require 'scss_lint/rake_task' + + SCSSLint::RakeTask.new do |t| + t.config = '.scss-lint.yml' + # See https://github.com/brigade/scss-lint/issues/726 + # Hack, otherwise linter won't respect scss_files option in config file. + t.files = [] + end +end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 0985ef3a669..2cf7a25a0fd 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -46,20 +46,11 @@ namespace :spec do run_commands(cmds) end - desc 'GitLab | Rspec | Run benchmark specs' - task :benchmark do - cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @benchmark) - ] - run_commands(cmds) - end - desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services --tag ~@benchmark) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) ] run_commands(cmds) end @@ -69,7 +60,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@benchmark), + %W(rspec spec), ] run_commands(cmds) end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 3acfc6e2075..01d23b89bb7 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -4,53 +4,59 @@ namespace :spinach do namespace :project do desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features" task :half do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags @project_commits,@project_issues,@project_merge_requests), - ] - run_commands(cmds) + run_spinach_tests('@project_commits,@project_issues,@project_merge_requests') end desc "GitLab | Spinach | Run remaining project spinach features" task :rest do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests), - ] - run_commands(cmds) + run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests') end end desc "GitLab | Spinach | Run project spinach features" task :project do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets), - ] - run_commands(cmds) + run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets') end desc "GitLab | Spinach | Run other spinach features" task :other do - cmds = [ - %W(rake gitlab:setup), - %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets), - ] - run_commands(cmds) + run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets') + end + + desc "GitLab | Spinach | Run other spinach features" + task :builds do + run_spinach_tests('@builds') end end desc "GitLab | Run spinach" task :spinach do - cmds = [ - %W(rake gitlab:setup), - %W(spinach), - ] - run_commands(cmds) + run_spinach_tests(nil) +end + +def run_command(cmd) + system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) end -def run_commands(cmds) - cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") +def run_spinach_command(args) + run_command(%w(spinach -r rerun) + args) +end + +def run_spinach_tests(tags) + #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!') + + success = run_spinach_command(%W(--tags #{tags})) + 3.times do |_| + break if success + break unless File.exists?('tmp/spinach-rerun.txt') + + tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp) + puts '' + puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red + puts '' + sleep(3) + success = run_spinach_command(tests) end + + raise("spinach tests for #{tags} failed!") unless success end diff --git a/lib/version_check.rb b/lib/version_check.rb index ea23344948c..91ad07feee5 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -13,6 +13,6 @@ class VersionCheck end def host - 'https://version.gitlab.com/check.png' + 'https://version.gitlab.com/check.svg' end end |