diff options
| author | Jan-Gerd Tenberge <janten@gmail.com> | 2015-10-22 21:43:17 +0200 |
|---|---|---|
| committer | Jan-Gerd Tenberge <janten@gmail.com> | 2015-10-22 21:43:17 +0200 |
| commit | b5c19bcc4d02fd498bc95d70c39bd88ec9cdda4b (patch) | |
| tree | 6eaed47ca3b65dbd622813497b18902666d23d68 /lib | |
| parent | b32bb377993fb0224ed3bd9294d752d1a82b2ef9 (diff) | |
| parent | a7174efaec77e9408900336b5941d9cca1f82ccf (diff) | |
| download | gitlab-ce-b5c19bcc4d02fd498bc95d70c39bd88ec9cdda4b.tar.gz | |
Fix merge error
Diffstat (limited to 'lib')
58 files changed, 740 insertions, 578 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c09488d3547..afc0402f9e1 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -46,6 +46,7 @@ module API mount Services mount Files mount Commits + mount CommitStatus mount Namespaces mount Branches mount Labels diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb new file mode 100644 index 00000000000..2c0596c9dfb --- /dev/null +++ b/lib/api/commit_statuses.rb @@ -0,0 +1,80 @@ +require 'mime/types' + +module API + # Project commit statuses API + class CommitStatus < Grape::API + resource :projects do + before { authenticate! } + + # Get a commit's statuses + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # ref (optional) - The ref + # stage (optional) - The stage + # name (optional) - The name + # all (optional) - Show all statuses, default: false + # Examples: + # GET /projects/:id/repository/commits/:sha/statuses + get ':id/repository/commits/:sha/statuses' do + authorize! :read_commit_statuses, user_project + sha = params[:sha] + ci_commit = user_project.ci_commit(sha) + not_found! 'Commit' unless ci_commit + statuses = ci_commit.statuses + statuses = statuses.latest unless parse_boolean(params[:all]) + statuses = statuses.where(ref: params[:ref]) if params[:ref].present? + statuses = statuses.where(stage: params[:stage]) if params[:stage].present? + statuses = statuses.where(name: params[:name]) if params[:name].present? + present paginate(statuses), with: Entities::CommitStatus + end + + # Post status to commit + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The commit hash + # ref (optional) - The ref + # state (required) - The state of the status. Can be: pending, running, success, error or failure + # target_url (optional) - The target URL to associate with this status + # description (optional) - A short description of the status + # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" + # Examples: + # POST /projects/:id/statuses/:sha + post ':id/statuses/:sha' do + authorize! :create_commit_status, user_project + required_attributes! [:state] + attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + commit = @project.commit(params[:sha]) + not_found! 'Commit' unless commit + + ci_commit = @project.ensure_ci_commit(commit.sha) + + name = params[:name] || params[:context] + status = GenericCommitStatus.running_or_pending.find_by(commit: ci_commit, name: name, ref: params[:ref]) + status ||= GenericCommitStatus.new(commit: ci_commit, user: current_user) + status.update(attrs) + + case params[:state].to_s + when 'running' + status.run + when 'success' + status.success + when 'failed' + status.drop + when 'canceled' + status.cancel + else + status.status = params[:state].to_s + end + + if status.save + present status, with: Entities::CommitStatus + else + render_validation_error!(status) + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9620d36ac41..883a5e14b17 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -149,6 +149,7 @@ module API class RepoCommitDetail < RepoCommit expose :parent_ids, :committed_date, :authored_date + expose :status end class ProjectSnippet < Grape::Entity @@ -228,6 +229,12 @@ module API expose :created_at end + class CommitStatus < Grape::Entity + expose :id, :sha, :ref, :status, :name, :target_url, :description, + :created_at, :started_at, :finished_at + expose :author, using: Entities::UserBasic + end + class Event < Grape::Entity expose :title, :project_id, :action_name expose :target_id, :target_type, :author_id @@ -255,6 +262,18 @@ module API expose :notification_level end + class ProjectService < Grape::Entity + expose :id, :title, :created_at, :updated_at, :active + expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events + # Expose serialized properties + expose :properties do |service, options| + field_names = service.fields. + select { |field| options[:include_passwords] || field[:type] != 'password' }. + map { |field| field[:name] } + service.properties.slice(*field_names) + end + end + class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 7fada98fcdc..549b1f9e9a7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -63,11 +63,11 @@ module API user_project.build_missing_services service_method = "#{underscored_service}_service" - + send_service(service_method) end end - + @project_service || not_found!("Service") end @@ -149,7 +149,6 @@ module API end def attributes_for_keys(keys, custom_params = nil) - params_hash = custom_params || params attrs = {} keys.each do |key| if params[key].present? or (params.has_key?(key) and params[key] == false) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 63ea2f05438..6eb84baf9cb 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -99,7 +99,7 @@ module API # id (required) - The ID of a project - this will be the source of the merge request # source_branch (required) - The source branch # target_branch (required) - The target branch - # target_project - The target project of the merge request defaults to the :id of the project + # target_project_id - The target project of the merge request defaults to the :id of the project # assignee_id - Assignee user ID # title (required) - Title of MR # description - Description of MR @@ -249,8 +249,16 @@ module API required_attributes! [:note] merge_request = user_project.merge_requests.find(params[:merge_request_id]) - note = merge_request.notes.new(note: params[:note], project_id: user_project.id) - note.author = current_user + + authorize! :create_note, merge_request + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute if note.save present note, with: Entities::MRNote diff --git a/lib/api/projects.rb b/lib/api/projects.rb index c2fb36b4143..67ee66a2058 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -246,8 +246,8 @@ module API # Example Request: # DELETE /projects/:id/fork delete ":id/fork" do - authenticated_as_admin! - unless user_project.forked_project_link.nil? + authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2d96c9666d2..20d568cf462 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -133,7 +133,7 @@ module API authorize! :download_code, user_project begin - file_path = ArchiveRepositoryService.new( + ArchiveRepositoryService.new( user_project, params[:sha], params[:format] @@ -141,17 +141,6 @@ module API rescue not_found!('File') end - - if file_path && File.exists?(file_path) - data = File.open(file_path, 'rb').read - basename = File.basename(file_path) - header['Content-Disposition'] = "attachment; filename=\"#{basename}\"" - content_type MIME::Types.type_for(file_path).first.content_type - env['api.format'] = :binary - present data - else - redirect request.fullpath - end end # Compare two branches, tags or commits diff --git a/lib/api/services.rb b/lib/api/services.rb index 6727e80ac1e..203f04a6259 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -57,7 +57,7 @@ module API # GET /project/:id/services/gitlab-ci # get ':id/services/:service_slug' do - present project_service + present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index ac63f89c6ec..5c42f25f4a2 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -45,7 +45,8 @@ module Backup directory = connection.directories.get(remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size) + multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, + encryption: Gitlab.config.backup.upload.encryption) $progress.puts "done".green else puts "uploading backup to #{remote_directory} failed".red @@ -55,7 +56,7 @@ module Backup def cleanup $progress.print "Deleting tmp directories ... " - + backup_contents.each do |dir| next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) @@ -75,7 +76,7 @@ module Backup if keep_time > 0 removed = 0 - + Dir.chdir(Gitlab.config.backup.path) do file_list = Dir.glob('*_gitlab_backup.tar') file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 5109c84e0ea..218d8c3adcc 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -23,10 +23,6 @@ module Ci rack_response({ 'message' => '500 Internal Server Error' }, 500) end - before do - check_enable_flag! - end - format :json helpers Helpers diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb index bac463a5909..a60769d8305 100644 --- a/lib/ci/api/commits.rb +++ b/lib/ci/api/commits.rb @@ -51,7 +51,7 @@ module Ci required_attributes! [:project_id, :data, :project_token] project = Ci::Project.find(params[:project_id]) authenticate_project_token!(project) - commit = Ci::CreateCommitService.new.execute(project, params[:data]) + commit = Ci::CreateCommitService.new.execute(project, current_user, params[:data]) if commit.persisted? present commit, with: Entities::CommitWithBuilds diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index f47bc1236b8..b80c0b8b273 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -2,7 +2,7 @@ module Ci module API module Entities class Commit < Grape::Entity - expose :id, :ref, :sha, :project_id, :before_sha, :created_at + expose :id, :sha, :project_id, :created_at expose :status, :finished_at, :duration expose :git_commit_message, :git_author_name, :git_author_email end @@ -12,7 +12,7 @@ module Ci end class Build < Grape::Entity - expose :id, :commands, :ref, :sha, :project_id, :repo_url, + expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url, :before_sha, :allow_git_fetch, :project_name expose :options do |model| diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 8e893aa5cc6..e602cda81d6 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,12 +3,6 @@ module Ci module Helpers UPDATE_RUNNER_EVERY = 60 - def check_enable_flag! - unless current_application_settings.ci_enabled - render_api_error!('400 (Bad request) CI is disabled', 400) - end - end - def authenticate_runners! forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN end diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb index 66bcf65e8c4..d719ad9e8d5 100644 --- a/lib/ci/api/projects.rb +++ b/lib/ci/api/projects.rb @@ -75,23 +75,17 @@ module Ci # Create Gitlab CI project using Gitlab project info # # Parameters: - # name (required) - The name of the project # gitlab_id (required) - The gitlab id of the project - # path (required) - The gitlab project path, ex. randx/six - # ssh_url_to_repo (required) - The gitlab ssh url to the repo # default_ref - The branch to run against (defaults to `master`) # Example Request: # POST /projects post do - required_attributes! [:name, :gitlab_id, :ssh_url_to_repo] + required_attributes! [:gitlab_id] filtered_params = { - name: params[:name], gitlab_id: params[:gitlab_id], # we accept gitlab_url for backward compatibility for a while (added to 7.11) - path: params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'), - default_ref: params[:default_ref] || 'master', - ssh_url_to_repo: params[:ssh_url_to_repo] + default_ref: params[:default_ref] || 'master' } project = Ci::Project.new(filtered_params) @@ -109,11 +103,7 @@ module Ci # # Parameters: # id (required) - The ID of a project - # name - The name of the project - # gitlab_id - The gitlab id of the project - # path - The gitlab project path, ex. randx/six - # ssh_url_to_repo - The gitlab ssh url to the repo - # default_ref - The branch to run against (defaults to `master`) + # default_ref - The branch to run against (defaults to `master`) # Example Request: # PUT /projects/:id put ":id" do @@ -121,12 +111,7 @@ module Ci unauthorized! unless can?(current_user, :admin_project, project.gl_project) - attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] - - # we accept gitlab_url for backward compatibility for a while (added to 7.11) - if attrs[:gitlab_url] && !attrs[:path] - attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1') - end + attrs = attributes_for_keys [:default_ref] if project.update_attributes(attrs) present project, with: Entities::Project diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e625e790df8..efcd2faffc7 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -5,7 +5,7 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] - ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when] attr_reader :before_script, :image, :services, :variables @@ -85,13 +85,15 @@ module Ci def build_job(name, job) { + stage_idx: stages.index(job[:stage]), stage: job[:stage], - script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", - tags: job[:tags] || [], + commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + tag_list: job[:tags] || [], name: name, only: job[:only], except: job[:except], allow_failure: job[:allow_failure] || false, + when: job[:when] || 'on_success', options: { image: job[:image] || @image, services: job[:services] || @services @@ -137,62 +139,74 @@ module Ci end @jobs.each do |name, job| - validate_job!("#{name} job", job) + validate_job!(name, job) end true end def validate_job!(name, job) + if name.blank? || !validate_string(name) + raise ValidationError, "job name should be non-empty string" + end + job.keys.each do |key| unless ALLOWED_JOB_KEYS.include? key - raise ValidationError, "#{name}: unknown parameter #{key}" + raise ValidationError, "#{name} job: unknown parameter #{key}" end end - if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name}: script should be a string or an array of a strings" + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" end if job[:stage] unless job[:stage].is_a?(String) && job[:stage].in?(stages) - raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}" + raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" end end - if job[:image] && !job[:image].is_a?(String) - raise ValidationError, "#{name}: image should be a string" + if job[:image] && !validate_string(job[:image]) + raise ValidationError, "#{name} job: image should be a string" end if job[:services] && !validate_array_of_strings(job[:services]) - raise ValidationError, "#{name}: services should be an array of strings" + raise ValidationError, "#{name} job: services should be an array of strings" end if job[:tags] && !validate_array_of_strings(job[:tags]) - raise ValidationError, "#{name}: tags parameter should be an array of strings" + raise ValidationError, "#{name} job: tags parameter should be an array of strings" end if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name}: only parameter should be an array of strings" + raise ValidationError, "#{name} job: only parameter should be an array of strings" end if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name}: except parameter should be an array of strings" + raise ValidationError, "#{name} job: except parameter should be an array of strings" end if job[:allow_failure] && !job[:allow_failure].in?([true, false]) - raise ValidationError, "#{name}: allow_failure parameter should be an boolean" + raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" + end + + if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end end private def validate_array_of_strings(values) - values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} + values.is_a?(Array) && values.all? { |value| validate_string(value) } end def validate_variables(variables) - variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) end end end diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb deleted file mode 100644 index c4f62e55295..00000000000 --- a/lib/ci/migrate/builds.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Ci - module Migrate - class Builds - attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir - - def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz') - end - - def restore - backup_existing_builds_dir - - FileUtils.mkdir_p(app_builds_dir, mode: 0700) - unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball) - abort 'Restore failed'.red - end - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end - end - end - end -end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb deleted file mode 100644 index bf9b80f1f62..00000000000 --- a/lib/ci/migrate/database.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Database - attr_reader :config - - def initialize - @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] - end - - def restore - decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) - decompress_wr.close - - restore_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - pg_env - spawn('psql', config['database'], in: decompress_rd) - end - decompress_rd.close - - success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } - abort 'Restore failed' unless success - end - - protected - - def db_file_name - File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') - end - - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - - def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] - end - - def report_success(success) - if success - puts '[DONE]'.green - else - puts '[FAILED]'.red - end - end - end - end -end diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb deleted file mode 100644 index e5e4fb784eb..00000000000 --- a/lib/ci/migrate/manager.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Ci - module Migrate - class Manager - CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x - - def cleanup - $progress.print "Deleting tmp directories ... " - - backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) - - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green - else - puts "deleting tmp directory '#{dir}' failed".red - abort 'Backup failed' - end - end - end - - def unpack - Dir.chdir(Gitlab.config.backup.path) - - # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 - - if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" - exit 1 - end - - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") - - unless File.exists?(tar_file) - puts "The specified CI backup doesn't exist!" - exit 1 - end - - $progress.print "Unpacking backup ... " - - unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red - exit 1 - else - $progress.puts "done".green - end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX) - puts "GitLab CI version mismatch:".red - puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red - exit 1 - end - end - - private - - def backup_contents - ["db", "builds", "backup_information.yml"] - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") - end - end - end -end - diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb deleted file mode 100644 index 97e043ece27..00000000000 --- a/lib/ci/migrate/tags.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'yaml' - -module Ci - module Migrate - class Tags - def restore - puts 'Inserting tags...' - connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag| - begin - connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})") - rescue ActiveRecord::RecordNotUnique - end - end - - ActiveRecord::Base.transaction do - puts 'Deleting old taggings...' - connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'" - - puts 'Inserting taggings...' - connection.execute( - 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' + - "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " + - 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' + - 'JOIN tags ON tags.name = ci_tags.name ' - ) - - puts 'Resetting counters... ' - connection.execute( - 'UPDATE tags SET ' + - 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)' - ) - end - end - - protected - - def connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/ci/status.rb b/lib/ci/status.rb new file mode 100644 index 00000000000..c02b3b8f3e4 --- /dev/null +++ b/lib/ci/status.rb @@ -0,0 +1,21 @@ +module Ci + class Status + def self.get_status(statuses) + statuses.reject! { |status| status.try(&:allow_failure?) } + + if statuses.none? + 'skipped' + elsif statuses.all?(&:success?) + 'success' + elsif statuses.all?(&:pending?) + 'pending' + elsif statuses.any?(&:running?) || statuses.any?(&:pending?) + 'running' + elsif statuses.all?(&:canceled?) + 'canceled' + else + 'failed' + end + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 163937c02cf..f15b2cfd231 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -47,7 +47,7 @@ class EventFilter actions << Event::COMMENTED if filter.include? 'comments' - events = events.where(action: actions) + events.where(action: actions) end def options(key) diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 322aed5e27c..51e46da82cc 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -110,7 +110,7 @@ module ExtractsPath @project, @ref, @path) rescue RuntimeError, NoMethodError, InvalidPathError - not_found! + render_404 end def tree diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 0353b3b7ed3..6830a916bcb 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -193,7 +193,14 @@ module Grack end def render_grack_auth_ok - [200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user) })]] + [ + 200, + { "Content-Type" => "application/json" }, + [JSON.dump({ + 'GL_ID' => Gitlab::ShellEnv.gl_id(@user), + 'RepoPath' => project.repository.path_to_repo, + })] + ] end def render_not_found diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 14ee4701e7b..01b8bda05c6 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -4,7 +4,8 @@ module Gitlab class KeyAdder < Struct.new(:io) def add_key(id, key) - io.puts("#{id}\t#{key.strip}") + key.gsub!(/[[:space:]]+/, ' ').strip! + io.puts("#{id}\t#{key}") end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 45bb904ed7a..8a7f8dc5003 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -12,7 +12,6 @@ module Gitlab @timestamps = {} date_from = 1.year.ago - date_to = Date.today events = Event.reorder(nil).contributions.where(author_id: user.id). where("created_at > ?", date_from).where(project_id: projects). diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb new file mode 100644 index 00000000000..71f37f1fef8 --- /dev/null +++ b/lib/gitlab/database.rb @@ -0,0 +1,11 @@ +module Gitlab + module Database + def self.mysql? + ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' + end + + def self.postgresql? + ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 4daf65331e8..142058aa69d 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,6 +44,14 @@ module Gitlab diff.old_path end end + + def added_lines + diff_lines.select(&:added?).size + end + + def removed_lines + diff_lines.select(&:removed?).size + end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 8ac1b15e88a..0072194606e 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -7,6 +7,14 @@ module Gitlab @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos end + + def added? + type == 'new' + end + + def removed? + type == 'old' + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index c1d9520ddf1..7015fe36c3d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -14,8 +14,6 @@ module Gitlab lines_arr = ::Gitlab::InlineDiff.processing lines lines_arr.each do |line| - raw_line = line.dup - next if filename?(line) full_line = html_escape(line.gsub(/\n/, '')) diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index f02ea43910f..8b1b6f48ed5 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -23,7 +23,7 @@ module Gitlab import_url: Project::UNKNOWN_IMPORT_URL ).execute - import_data = project.create_import_data( + project.create_import_data( data: { 'repo' => repo.raw_data, 'user_map' => user_map, diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 0cfeaf9d61c..1cb7d16aeb3 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -23,7 +23,7 @@ module Gitlab import_url: repo.import_url ).execute - import_data = project.create_import_data( + project.create_import_data( data: { "repo" => repo.raw_data, "user_map" => user_map diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 856ccc71084..9068d79c95e 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -24,12 +24,12 @@ module Gitlab match[1] end - private - def config Gitlab.config.incoming_email end + private + def address_regex wildcard_address = config.address return nil unless wildcard_address diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index cb66fd500fe..4be99dd88c2 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -14,7 +14,7 @@ module Gitlab # LDAP distinguished name is case-insensitive identity = ::Identity. where(provider: provider). - where('lower(extern_uid) = ?', uid.downcase).last + where('lower(extern_uid) = ?', uid.mb_chars.downcase.to_s).last identity && identity.user end end @@ -35,7 +35,7 @@ module Gitlab end def find_by_email - ::User.find_by(email: auth_hash.email) + ::User.find_by(email: auth_hash.email.downcase) end def update_user_attributes diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index ae5f2544691..b082bfc434b 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -7,6 +7,14 @@ module Gitlab module Markdown # Convert a Markdown String into an HTML-safe String of HTML # + # Note that while the returned HTML will have been sanitized of dangerous + # HTML, it may post a risk of information leakage if it's not also passed + # through `post_process`. + # + # Also note that the returned String is always HTML, not XHTML. Views + # requiring XHTML, such as Atom feeds, need to call `post_process` on the + # result, providing the appropriate `pipeline` option. + # # markdown - Markdown String # context - Hash of context options passed to our HTML Pipeline # @@ -31,6 +39,33 @@ module Gitlab renderer.render(markdown) end + # Perform post-processing on an HTML String + # + # This method is used to perform state-dependent changes to a String of + # HTML, such as removing references that the current user doesn't have + # permission to make (`RedactorFilter`). + # + # html - String to process + # options - Hash of options to customize output + # :pipeline - Symbol pipeline type + # :project - Project + # :user - User object + # + # Returns an HTML-safe String + def self.post_process(html, options) + context = { + project: options[:project], + current_user: options[:user] + } + doc = post_processor.to_document(html, context) + + if options[:pipeline] == :atom + doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML) + else + doc.to_html + end.html_safe + end + # Provide autoload paths for filters to prevent a circular dependency error autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' @@ -41,6 +76,7 @@ module Gitlab autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter' autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter' autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter' + autoload :RedactorFilter, 'gitlab/markdown/redactor_filter' autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' @@ -48,27 +84,22 @@ module Gitlab autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' + autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter' - # Public: Parse the provided text with GitLab-Flavored Markdown + # Public: Parse the provided HTML with GitLab-Flavored Markdown + # + # html - HTML String + # options - A Hash of options used to customize output (default: {}) + # :no_header_anchors - Disable header anchors in TableOfContentsFilter + # :path - Current path String + # :pipeline - Symbol pipeline type + # :project - Current Project object + # :project_wiki - Current ProjectWiki object + # :ref - Current ref String # - # text - the source text - # options - A Hash of options used to customize output (default: {}): - # :xhtml - output XHTML instead of HTML - # :reference_only_path - Use relative path for reference links - def self.gfm(text, options = {}) - return text if text.nil? - - # Duplicate the string so we don't alter the original, then call to_str - # to cast it back to a String instead of a SafeBuffer. This is required - # for gsub calls to work as we need them to. - text = text.dup.to_str - - options.reverse_merge!( - xhtml: false, - reference_only_path: true, - project: options[:project], - current_user: options[:current_user] - ) + # Returns an HTML-safe String + def self.gfm(html, options = {}) + return '' unless html.present? @pipeline ||= HTML::Pipeline.new(filters) @@ -77,41 +108,36 @@ module Gitlab pipeline: options[:pipeline], # EmojiFilter - asset_root: Gitlab.config.gitlab.base_url, asset_host: Gitlab::Application.config.asset_host, - - # TableOfContentsFilter - no_header_anchors: options[:no_header_anchors], + asset_root: Gitlab.config.gitlab.base_url, # ReferenceFilter - current_user: options[:current_user], - only_path: options[:reference_only_path], - project: options[:project], + only_path: only_path_pipeline?(options[:pipeline]), + project: options[:project], # RelativeLinkFilter + project_wiki: options[:project_wiki], ref: options[:ref], requested_path: options[:path], - project_wiki: options[:project_wiki] - } - - result = @pipeline.call(text, context) - save_options = 0 - if options[:xhtml] - save_options |= Nokogiri::XML::Node::SaveOptions::AS_XHTML - end - - text = result[:output].to_html(save_with: save_options) + # TableOfContentsFilter + no_header_anchors: options[:no_header_anchors] + } - text.html_safe + @pipeline.to_html(html, context).html_safe end private - def self.renderer - @markdown ||= begin - renderer = Redcarpet::Render::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + # Check if a pipeline enables the `only_path` context option + # + # Returns Boolean + def self.only_path_pipeline?(pipeline) + case pipeline + when :atom, :email + false + else + true end end @@ -129,6 +155,17 @@ module Gitlab }.freeze end + def self.renderer + @markdown ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + + def self.post_processor + @post_processor ||= HTML::Pipeline.new([Gitlab::Markdown::RedactorFilter]) + end + # Filters used in our pipeline # # SanitizationFilter should come first so that all generated reference HTML @@ -140,6 +177,7 @@ module Gitlab Gitlab::Markdown::SyntaxHighlightFilter, Gitlab::Markdown::SanitizationFilter, + Gitlab::Markdown::UploadLinkFilter, Gitlab::Markdown::RelativeLinkFilter, Gitlab::Markdown::EmojiFilter, Gitlab::Markdown::TableOfContentsFilter, diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index bb496135d92..e070edae0a4 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -26,6 +26,18 @@ module Gitlab end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit-range") + range = CommitRange.new(id, project) + + return unless range.valid_commits? + + { commit_range: range } + end + def initialize(*args) super @@ -53,13 +65,11 @@ module Gitlab range = CommitRange.new(id, project) if range.valid_commits? - push_result(:commit_range, range) - url = url_for_commit_range(project, range) title = range.reference_title klass = reference_class(:commit_range) - data = data_attribute(project.id) + data = data_attribute(project: project.id, commit_range: id) project_ref += '@' if project_ref diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index fcbb2e936a5..8cdbeb1f9cf 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -26,6 +26,18 @@ module Gitlab end end + def self.referenced_by(node) + project = Project.find(node.attr("data-project")) rescue nil + return unless project + + id = node.attr("data-commit") + commit = commit_from_ref(project, id) + + return unless commit + + { commit: commit } + end + def call replace_text_nodes_matching(Commit.reference_pattern) do |content| commit_link_filter(content) @@ -39,17 +51,15 @@ module Gitlab # Returns a String with commit references replaced with links. All links # have `gfm` and `gfm-commit` class names attached for styling. def commit_link_filter(text) - self.class.references_in(text) do |match, commit_ref, project_ref| + self.class.references_in(text) do |match, id, project_ref| project = self.project_from_ref(project_ref) - if commit = commit_from_ref(project, commit_ref) - push_result(:commit, commit) - + if commit = self.class.commit_from_ref(project, id) url = url_for_commit(project, commit) title = escape_once(commit.link_title) klass = reference_class(:commit) - data = data_attribute(project.id) + data = data_attribute(project: project.id, commit: id) project_ref += '@' if project_ref @@ -62,9 +72,9 @@ module Gitlab end end - def commit_from_ref(project, commit_ref) + def self.commit_from_ref(project, id) if project && project.valid_repo? - project.commit(commit_ref) + project.commit(id) end end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb index 855748fdccc..6ab04a584b0 100644 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ b/lib/gitlab/markdown/cross_project_reference.rb @@ -13,18 +13,11 @@ module Gitlab # # ref - String reference. # - # Returns a Project, or nil if the reference can't be accessed + # Returns a Project, or nil if the reference can't be found def project_from_ref(ref) return context[:project] unless ref - other = Project.find_with_namespace(ref) - return nil unless other && user_can_reference_project?(other) - - other - end - - def user_can_reference_project?(project, user = context[:current_user]) - Ability.abilities.allowed?(user, :read_project, project) + Project.find_with_namespace(ref) end end end diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb index f7c43e1ca89..8f86f13976a 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/gitlab/markdown/external_issue_reference_filter.rb @@ -47,8 +47,9 @@ module Gitlab title = escape_once("Issue in #{project.external_issue_tracker.title}") klass = reference_class(:issue) + data = data_attribute(project: project.id) - %(<a href="#{url}" + %(<a href="#{url}" #{data} title="#{title}" class="#{klass}">#{match}</a>) end diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index 01320f80796..481d282f7b1 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { issue: LazyReference.new(Issue, node.attr("data-issue")) } + end + def call replace_text_nodes_matching(Issue.reference_pattern) do |content| issue_link_filter(content) @@ -45,13 +49,11 @@ module Gitlab project = self.project_from_ref(project_ref) if project && issue = project.get_issue(id) - push_result(:issue, issue) - url = url_for_issue(id, project, only_path: context[:only_path]) title = escape_once("Issue: #{issue.title}") klass = reference_class(:issue) - data = data_attribute(project.id) + data = data_attribute(project: project.id, issue: issue.id) %(<a href="#{url}" #{data} title="#{title}" diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 1e5cb12071e..618acb7a578 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -22,6 +22,10 @@ module Gitlab end end + def self.referenced_by(node) + { label: LazyReference.new(Label, node.attr("data-label")) } + end + def call replace_text_nodes_matching(Label.reference_pattern) do |content| label_link_filter(content) @@ -41,11 +45,9 @@ module Gitlab params = label_params(id, name) if label = project.labels.find_by(params) - push_result(:label, label) - url = url_for_label(project, label) klass = reference_class(:label) - data = data_attribute(project.id) + data = data_attribute(project: project.id, label: label.id) %(<a href="#{url}" #{data} class="#{klass}">#{render_colored_label(label)}</a>) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index ecbd263d0e0..5bc63269808 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { merge_request: LazyReference.new(MergeRequest, node.attr("data-merge-request")) } + end + def call replace_text_nodes_matching(MergeRequest.reference_pattern) do |content| merge_request_link_filter(content) @@ -45,11 +49,9 @@ module Gitlab project = self.project_from_ref(project_ref) if project && merge_request = project.merge_requests.find_by(iid: id) - push_result(:merge_request, merge_request) - title = escape_once("Merge Request: #{merge_request.title}") klass = reference_class(:merge_request) - data = data_attribute(project.id) + data = data_attribute(project: project.id, merge_request: merge_request.id) url = url_for_merge_request(merge_request, project) diff --git a/lib/gitlab/markdown/redactor_filter.rb b/lib/gitlab/markdown/redactor_filter.rb new file mode 100644 index 00000000000..a1f3a8a8ebf --- /dev/null +++ b/lib/gitlab/markdown/redactor_filter.rb @@ -0,0 +1,40 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that removes references to records that the current user does + # not have permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class RedactorFilter < HTML::Pipeline::Filter + def call + doc.css('a.gfm').each do |node| + unless user_can_reference?(node) + node.replace(node.text) + end + end + + doc + end + + private + + def user_can_reference?(node) + if node.has_attribute?('data-reference-filter') + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + reference_filter.user_can_reference?(current_user, node, context) + else + true + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb index 9b293c957d6..a4c560f578c 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/gitlab/markdown/reference_filter.rb @@ -11,30 +11,57 @@ module Gitlab # Context options: # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. - # - # Results: - # :references - A Hash of references that were found and replaced. class ReferenceFilter < HTML::Pipeline::Filter - def initialize(*args) - super + LazyReference = Struct.new(:klass, :ids) do + def self.load(refs) + lazy_references, values = refs.partition { |ref| ref.is_a?(self) } + + lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs| + ids = refs.flat_map(&:ids) + klass.where(id: ids) + end + + values + lazy_values + end + + def load + self.klass.where(id: self.ids) + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-project') + project_id = node.attr('data-project').to_i + return true if project_id == context[:project].try(:id) + + project = Project.find(project_id) rescue nil + Ability.abilities.allowed?(user, :read_project, project) + else + true + end + end - result[:references] = Hash.new { |hash, type| hash[type] = [] } + def self.referenced_by(node) + raise NotImplementedError, "#{self} does not implement #{__method__}" end # Returns a data attribute String to attach to a reference link # - # id - Object ID - # type - Object type (default: :project) + # attributes - Hash, where the key becomes the data attribute name and the + # value is the data attribute value # # Examples: # - # data_attribute(1) # => "data-project-id=\"1\"" - # data_attribute(2, :user) # => "data-user-id=\"2\"" - # data_attribute(3, :group) # => "data-group-id=\"3\"" + # data_attribute(project: 1, issue: 2) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # + # data_attribute(project: 3, merge_request: 4) + # # => "data-reference-filter=\"Gitlab::Markdown::SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String - def data_attribute(id, type = :project) - %Q(data-#{type}-id="#{id}") + def data_attribute(attributes = {}) + attributes[:reference_filter] = self.class.name + attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{value}") }.join(" ") end def escape_once(html) @@ -59,16 +86,6 @@ module Gitlab context[:project] end - # Add a reference to the pipeline's result Hash - # - # type - Singular Symbol reference type (e.g., :issue, :user, etc.) - # values - One or more Objects to add - def push_result(type, *values) - return if values.empty? - - result[:references][type].push(*values) - end - def reference_class(type) "gfm gfm-#{type}" end @@ -85,15 +102,15 @@ module Gitlab # Yields the current node's String contents. The result of the block will # replace the node's existing content and update the current document. # - # Returns the updated Nokogiri::XML::Document object. + # Returns the updated Nokogiri::HTML::DocumentFragment object. def replace_text_nodes_matching(pattern) return doc if project.nil? search_text_nodes(doc).each do |node| - content = node.to_html - - next unless content.match(pattern) next if ignored_ancestry?(node) + next unless node.text =~ pattern + + content = node.to_html html = yield content diff --git a/lib/gitlab/markdown/reference_gatherer_filter.rb b/lib/gitlab/markdown/reference_gatherer_filter.rb new file mode 100644 index 00000000000..00f983675e6 --- /dev/null +++ b/lib/gitlab/markdown/reference_gatherer_filter.rb @@ -0,0 +1,63 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' + +module Gitlab + module Markdown + # HTML filter that gathers all referenced records that the current user has + # permission to view. + # + # Expected to be run in its own post-processing pipeline. + # + class ReferenceGathererFilter < HTML::Pipeline::Filter + def initialize(*) + super + + result[:references] ||= Hash.new { |hash, type| hash[type] = [] } + end + + def call + doc.css('a.gfm').each do |node| + gather_references(node) + end + + load_lazy_references unless context[:load_lazy_references] == false + + doc + end + + private + + def gather_references(node) + return unless node.has_attribute?('data-reference-filter') + + reference_type = node.attr('data-reference-filter') + reference_filter = reference_type.constantize + + return if context[:reference_filter] && reference_filter != context[:reference_filter] + + return unless reference_filter.user_can_reference?(current_user, node, context) + + references = reference_filter.referenced_by(node) + return unless references + + references.each do |type, values| + Array.wrap(values).each do |value| + result[:references][type] << value + end + end + end + + # Will load all references of one type using one query. + def load_lazy_references + refs = result[:references] + refs.each do |type, values| + refs[type] = ReferenceFilter::LazyReference.load(values) + end + end + + def current_user + context[:current_user] + end + end + end +end diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index e2cf89cb1d8..f783f951711 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -27,6 +27,10 @@ module Gitlab end end + def self.referenced_by(node) + { snippet: LazyReference.new(Snippet, node.attr("data-snippet")) } + end + def call replace_text_nodes_matching(Snippet.reference_pattern) do |content| snippet_link_filter(content) @@ -45,11 +49,9 @@ module Gitlab project = self.project_from_ref(project_ref) if project && snippet = project.snippets.find_by(id: id) - push_result(:snippet, snippet) - title = escape_once("Snippet: #{snippet.title}") klass = reference_class(:snippet) - data = data_attribute(project.id) + data = data_attribute(project: project.id, snippet: snippet.id) url = url_for_snippet(snippet, project) diff --git a/lib/gitlab/markdown/upload_link_filter.rb b/lib/gitlab/markdown/upload_link_filter.rb new file mode 100644 index 00000000000..fbada73ab86 --- /dev/null +++ b/lib/gitlab/markdown/upload_link_filter.rb @@ -0,0 +1,47 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' +require 'uri' + +module Gitlab + module Markdown + # HTML filter that "fixes" relative upload links to files. + # Context options: + # :project (required) - Current project + # + class UploadLinkFilter < HTML::Pipeline::Filter + def call + doc.search('a').each do |el| + process_link_attr el.attribute('href') + end + + doc.search('img').each do |el| + process_link_attr el.attribute('src') + end + + doc + end + + protected + + def process_link_attr(html_attr) + return if html_attr.blank? + + uri = html_attr.value + if uri.starts_with?("/uploads/") + html_attr.value = build_url(uri).to_s + end + end + + def build_url(uri) + File.join(Gitlab.config.gitlab.url, context[:project].path_with_namespace, uri) + end + + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project + end + end + end +end diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index 6f436ea7167..2a594e1662e 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -23,6 +23,31 @@ module Gitlab end end + def self.referenced_by(node) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + return unless group + + { user: group.users } + elsif node.has_attribute?('data-user') + { user: LazyReference.new(User, node.attr('data-user')) } + elsif node.has_attribute?('data-project') + project = Project.find(node.attr('data-project')) rescue nil + return unless project + + { user: project.team.members.flatten } + end + end + + def self.user_can_reference?(user, node, context) + if node.has_attribute?('data-group') + group = Group.find(node.attr('data-group')) rescue nil + Ability.abilities.allowed?(user, :read_group, group) + else + super + end + end + def call replace_text_nodes_matching(User.reference_pattern) do |content| user_link_filter(content) @@ -61,14 +86,12 @@ module Gitlab def link_to_all project = context[:project] - # FIXME (rspeicher): Law of Demeter - push_result(:user, *project.team.members.flatten) - url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) + data = data_attribute(project: project.id) text = User.reference_prefix + 'all' - %(<a href="#{url}" class="#{link_class}">#{text}</a>) + %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end def link_to_namespace(namespace) @@ -80,30 +103,20 @@ module Gitlab end def link_to_group(group, namespace) - return unless user_can_reference_group?(namespace) - - push_result(:user, *namespace.users) - url = urls.group_url(group, only_path: context[:only_path]) - data = data_attribute(namespace.id, :group) + data = data_attribute(group: namespace.id) text = Group.reference_prefix + group %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end def link_to_user(user, namespace) - push_result(:user, namespace.owner) - url = urls.user_url(user, only_path: context[:only_path]) - data = data_attribute(namespace.owner_id, :user) + data = data_attribute(user: namespace.owner_id) text = User.reference_prefix + user %(<a href="#{url}" #{data} class="#{link_class}">#{text}</a>) end - - def user_can_reference_group?(group) - Ability.abilities.allowed?(context[:current_user], :read_group, group) - end end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 0961bd80421..da8df8a3025 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,11 +3,12 @@ require 'gitlab/markdown' module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor - attr_accessor :project, :current_user + attr_accessor :project, :current_user, :load_lazy_references - def initialize(project, current_user = nil) + def initialize(project, current_user = nil, load_lazy_references: true) @project = project @current_user = current_user + @load_lazy_references = load_lazy_references end def analyze(text) @@ -26,9 +27,9 @@ module Gitlab def references @references ||= Hash.new do |references, type| type = type.to_sym - return references[type] if references.has_key?(type) + next references[type] if references.has_key?(type) - references[type] = pipeline_result(type).uniq + references[type] = pipeline_result(type) end end @@ -39,21 +40,34 @@ module Gitlab # # Returns the results Array for the requested filter type def pipeline_result(filter_type) - klass = filter_type.to_s.camelize + 'ReferenceFilter' + return [] if @text.blank? + + klass = "#{filter_type.to_s.camelize}ReferenceFilter" filter = Gitlab::Markdown.const_get(klass) context = { project: project, current_user: current_user, + # We don't actually care about the links generated only_path: true, - ignore_blockquotes: true + ignore_blockquotes: true, + + # ReferenceGathererFilter + load_lazy_references: false, + reference_filter: filter } - pipeline = HTML::Pipeline.new([filter], context) + pipeline = HTML::Pipeline.new([filter, Gitlab::Markdown::ReferenceGathererFilter], context) result = pipeline.call(@text) - result[:references][filter_type] + values = result[:references][filter_type].uniq + + if @load_lazy_references + values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq + end + + values end end end diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb new file mode 100644 index 00000000000..be8fcc7b2d2 --- /dev/null +++ b/lib/gitlab/uploads_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class UploadsTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + + def root_dir + File.join(Rails.root, "public", "uploads") + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 7218a4d2f20..1e55c5a0486 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -113,7 +113,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location @gitlab-git-http-server { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. # gzip off; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 7dabfba87e2..08641bbcc17 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -160,7 +160,25 @@ server { proxy_pass http://gitlab; } - location ~ [-\/\w\.]+\.git\/ { + location ~ ^/[\w\.-]+/[\w\.-]+/(info/refs|git-upload-pack|git-receive-pack)$ { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/[\w\.-]+/[\w\.-]+/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location ~ ^/api/v3/projects/.*/repository/archive { + # 'Error' 418 is a hack to re-use the @gitlab-git-http-server block + error_page 418 = @gitlab-git-http-server; + return 418; + } + + location @gitlab-git-http-server { ## If you use HTTPS make sure you disable gzip compression ## to be safe against BREACH attack. gzip off; diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake deleted file mode 100644 index 1de664c85e1..00000000000 --- a/lib/tasks/ci/migrate.rake +++ /dev/null @@ -1,87 +0,0 @@ -namespace :ci do - desc 'GitLab | Import and migrate CI database' - task migrate: :environment do - warn_user_is_not_gitlab - configure_cron_mode - - unless ENV['force'] == 'yes' - puts 'This will remove all CI related data and restore it from the provided backup.' - ask_to_continue - puts '' - end - - # disable CI for time of migration - enable_ci(false) - - # unpack archives - migrate = Ci::Migrate::Manager.new - migrate.unpack - - Rake::Task['ci:migrate:db'].invoke - Rake::Task['ci:migrate:builds'].invoke - Rake::Task['ci:migrate:tags'].invoke - Rake::Task['ci:migrate:services'].invoke - - # enable CI for time of migration - enable_ci(true) - - migrate.cleanup - end - - namespace :migrate do - desc 'GitLab | Import CI database' - task db: :environment do - configure_cron_mode - $progress.puts 'Restoring database ... '.blue - Ci::Migrate::Database.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Import CI builds' - task builds: :environment do - configure_cron_mode - $progress.puts 'Restoring builds ... '.blue - Ci::Migrate::Builds.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI tags' - task tags: :environment do - configure_cron_mode - $progress.puts 'Migrating tags ... '.blue - ::Ci::Migrate::Tags.new.restore - $progress.puts 'done'.green - end - - desc 'GitLab | Migrate CI auto-increments' - task autoincrements: :environment do - c = ActiveRecord::Base.connection - c.tables.select { |t| t.start_with?('ci_') }.each do |table| - result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1") - if result - ai_val = result['id'].to_i + 1 - puts "Resetting auto increment ID for #{table} to #{ai_val}" - if c.adapter_name == 'PostgreSQL' - c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}") - else - c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}") - end - end - end - end - - desc 'GitLab | Migrate CI services' - task services: :environment do - $progress.puts 'Migrating services ... '.blue - c = ActiveRecord::Base.connection - c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") - $progress.puts 'done'.green - end - end - - def enable_ci(enabled) - settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults - settings.ci_enabled = enabled - settings.save! - end -end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 66f1ecf385f..2e73f792a9d 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -335,7 +335,7 @@ namespace :gitlab do print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (.*)/) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) puts "yes".green @@ -642,7 +642,6 @@ namespace :gitlab do if Gitlab.config.incoming_email.enabled check_address_formatted_correctly - check_mail_room_config_exists check_imap_authentication if Rails.env.production? @@ -744,42 +743,16 @@ namespace :gitlab do end end - def check_mail_room_config_exists - print "MailRoom config exists? ... " - - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - if File.exists?(mail_room_config_file) - puts "yes".green - else - puts "no".red - try_fixing_it( - "Copy config/mail_room.yml.example to config/mail_room.yml", - "Check that the information in config/mail_room.yml is correct" - ) - for_more_information( - "doc/incoming_email/README.md" - ) - fix_and_rerun - end - end - def check_imap_authentication print "IMAP server credentials are correct? ... " - mail_room_config_file = Rails.root.join("config", "mail_room.yml") - - unless File.exists?(mail_room_config_file) - puts "can't check because of previous errors".magenta - return - end - - config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil + config = Gitlab.config.incoming_email if config begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.login(config[:email], config[:password]) + imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl) + imap.starttls if config.start_tls + imap.login(config.user, config.password) connected = true rescue connected = false @@ -791,7 +764,7 @@ namespace :gitlab do else puts "no".red try_fixing_it( - "Check that the information in config/mail_room.yml is correct" + "Check that the information in config/gitlab.yml is correct" ) for_more_information( "doc/incoming_email/README.md" diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 6b1e3716147..9f5852ac613 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -46,43 +46,24 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean repositories" task repos: :environment do warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] - - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') - - global_projects = Project.in_namespace(nil).pluck(:path) - - puts git_base_path.yellow - puts "Looking for global repos to remove... " - - # skip non git repo - all_dirs.select! do |dir| - dir =~ /.git$/ - end - - # skip existing repos - all_dirs.reject! do |dir| - repo_name = File.basename dir - path = repo_name.gsub(/\.git$/, "") - global_projects.include?(path) - end - all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".red - else - puts "Cannot remove #{dir_path}".red - end - else - puts "Can be removed: #{dir_path}".red + move_suffix = "+orphaned+#{Time.now.to_i}" + repo_root = Gitlab.config.gitlab_shell.repos_path + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) end end - - unless remove_flag - puts "To cleanup this directories run this command with REMOVE=true".yellow - end end desc "GitLab | Cleanup | Block users that have been removed in LDAP" diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 0ac4b0fa8a3..4cbccf2ca89 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -16,6 +16,7 @@ namespace :gitlab do Rake::Task["db:setup"].invoke Rake::Task["add_limits_mysql"].invoke + Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".red diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake new file mode 100644 index 00000000000..9196677a017 --- /dev/null +++ b/lib/tasks/gitlab/two_factor.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :two_factor do + desc "GitLab | Disable Two-factor authentication (2FA) for all users" + task disable_for_all_users: :environment do + scope = User.with_two_factor + count = scope.count + + if count > 0 + puts "This will disable 2FA for #{count.to_s.red} users..." + + begin + ask_to_continue + scope.find_each(&:disable_two_factor!) + puts "Successfully disabled 2FA for #{count} users.".green + rescue Gitlab::TaskAbortedByUserError + puts "Quitting...".red + end + else + puts "There are currently no users with 2FA enabled.".yellow + end + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake new file mode 100644 index 00000000000..141a0b74ec0 --- /dev/null +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -0,0 +1,8 @@ +require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') +require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') + +desc 'GitLab | Sets up PostgreSQL' +task setup_postgresql: :environment do + NamespacesProjectsPathLowerIndexes.new.up + AddUsersLowerUsernameEmailIndexes.new.up +end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 831746815d7..365ff2defd4 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -19,11 +19,20 @@ namespace :spec do run_commands(cmds) end + desc 'GitLab | Rspec | Run benchmark specs' + task :benchmark do + cmds = [ + %W(rake gitlab:setup), + %W(rspec spec --tag @benchmark) + ] + run_commands(cmds) + end + desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature) + %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark) ] run_commands(cmds) end @@ -33,7 +42,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %W(rake gitlab:setup), - %W(rspec spec), + %W(rspec spec --tag ~@benchmark), ] run_commands(cmds) end |