diff options
Diffstat (limited to 'lib')
35 files changed, 419 insertions, 379 deletions
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index c4fa1838b5a..2efe7e3adf3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -56,9 +56,9 @@ module API not_found!('Award Emoji') unless can_read_awardable? - award = awardable.award_emoji.new(name: params[:name], user: current_user) + award = awardable.create_award_emoji(params[:name], current_user) - if award.save + if award.persisted? present award, with: Entities::AwardEmoji else not_found!("Award Emoji #{award.errors.messages}") diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 323a7086890..acb4812b5cf 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -64,7 +64,7 @@ module API ref = branches.first end - pipeline = @project.ensure_pipeline(commit.sha, ref) + pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) name = params[:name] || params[:context] status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 301dbb688a7..3c79a00eb8c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -54,6 +54,7 @@ module API class BasicProjectDetails < Grape::Entity expose :id + expose :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace end @@ -186,6 +187,7 @@ module API end expose :user_notes_count expose :upvotes, :downvotes + expose :due_date end class ExternalIssue < Grape::Entity diff --git a/lib/api/internal.rb b/lib/api/internal.rb index d5dfba5e0cc..959b700de78 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -63,7 +63,12 @@ module API if access_status.status # Return the repository full path so that gitlab-shell has it when # handling ssh commands - response[:repository_path] = project.repository.path_to_repo + response[:repository_path] = + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end end response diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 8a03a41e9c5..c588103e517 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -152,12 +152,13 @@ module API # milestone_id (optional) - The ID of a milestone to assign issue # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # POST /projects/:id/issues - post ":id/issues" do + post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -201,12 +202,13 @@ module API # labels (optional) - The labels of an issue # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted + # due_date (optional) - Date time string in the format YEAR-MONTH-DAY # Example Request: # PUT /projects/:id/issues/:issue_id - put ":id/issues/:issue_id" do + put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0cc1edd65c8..6d2a6f3946c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -25,7 +25,11 @@ module API @projects = current_user.authorized_projects @projects = filter_projects(@projects) @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + if params[:simple] + present @projects, with: Entities::BasicProjectDetails, user: current_user + else + present @projects, with: Entities::ProjectWithAccess, user: current_user + end end # Get an owned projects list for authenticated user diff --git a/lib/banzai.rb b/lib/banzai.rb index 093382261ae..9ebe379f454 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.cache_collection_render(texts_and_contexts) + Renderer.cache_collection_render(texts_and_contexts) + end + def self.render_result(text, context = {}) Renderer.render_result(text, context) end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index fac7dad3243..9ed45707515 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -56,6 +56,8 @@ module Banzai # period (e.g., http://localhost:3000/) rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) + return if rinku == html + # Rinku returns a String, so parse it back to a Nokogiri::XML::Document # for further processing. @doc = parse_html(rinku) diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 536b478979f..91f0159f9a1 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -19,14 +19,22 @@ module Banzai language = node.attr('class') code = node.text + css_classes = "code highlight" + + lexer = Rouge::Lexer.find_fancy(language) || Rouge::Lexers::PlainText + formatter = Rouge::Formatters::HTML.new + begin - highlighted = block_code(code, language) + code = formatter.format(lexer.lex(code)) + + css_classes << " js-syntax-highlight #{lexer.tag}" rescue # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. - highlighted = "<pre>#{code}</pre>" end + highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) end @@ -40,8 +48,7 @@ module Banzai # Override Rouge::Plugins::Redcarpet#rouge_formatter def rouge_formatter(lexer) - Rouge::Formatters::HTMLGitlab.new( - cssclass: "code highlight js-syntax-highlight #{lexer.tag}") + Rouge::Formatters::HTML.new end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 5b0a6d8541b..e1ca7f4d24b 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -112,7 +112,7 @@ module Banzai data = data_attribute(project: project.id, author: author.try(:id)) text = link_text || User.reference_prefix + 'all' - link_tag(url, data, text) + link_tag(url, data, text, 'All Project and Group Members') end def link_to_namespace(namespace, link_text: nil) @@ -128,7 +128,7 @@ module Banzai data = data_attribute(group: namespace.id) text = link_text || Group.reference_prefix + group - link_tag(url, data, text) + link_tag(url, data, text, namespace.name) end def link_to_user(user, namespace, link_text: nil) @@ -136,11 +136,11 @@ module Banzai data = data_attribute(user: namespace.owner_id) text = link_text || User.reference_prefix + user - link_tag(url, data, text) + link_tag(url, data, text, namespace.owner_name) end - def link_tag(url, data, text) - %(<a href="#{url}" #{data} class="#{link_class}">#{escape_once(text)}</a>) + def link_tag(url, data, text, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index dc83b87a6c1..9aef807c152 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -39,9 +39,7 @@ module Banzai # Renders the attribute of every given object. def render_objects(objects, attribute) - objects.map do |object| - render_attribute(object, attribute) - end + render_attributes(objects, attribute) end # Redacts the list of documents. @@ -64,16 +62,21 @@ module Banzai context end - # Renders the attribute of an object. + # Renders the attributes of a set of objects. # - # Returns a `Nokogiri::HTML::Document`. - def render_attribute(object, attribute) - context = context_for(object, attribute) + # Returns an Array of `Nokogiri::HTML::Document`. + def render_attributes(objects, attribute) + strings_and_contexts = objects.map do |object| + context = context_for(object, attribute) + + string = object.__send__(attribute) - string = object.__send__(attribute) - html = Banzai.render(string, context) + { text: string, context: context } + end - Banzai::Pipeline[:relative_link].to_document(html, context) + Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| + Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + end end def base_context diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 6718acdef7e..910687a7b6a 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -10,7 +10,7 @@ module Banzai # requiring XHTML, such as Atom feeds, need to call `post_process` on the # result, providing the appropriate `pipeline` option. # - # markdown - Markdown String + # text - Markdown String # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String @@ -29,6 +29,58 @@ module Banzai end end + # Perform multiple render from an Array of Markdown String into an + # Array of HTML-safe String of HTML. + # + # As the rendered Markdown String can be already cached read all the data + # from the cache using Rails.cache.read_multi operation. If the Markdown String + # is not in the cache or it's not cacheable (no cache_key entry is provided in + # the context) the Markdown String is rendered and stored in the cache so the + # next render call gets the rendered HTML-safe String from the cache. + # + # For further explanation see #render method comments. + # + # texts_and_contexts - An Array of Hashes that contains the Markdown String (:text) + # an options passed to our HTML Pipeline (:context) + # + # If on the :context you specify a :cache_key entry will be used to retrieve it + # and cache the result of rendering the Markdown String. + # + # Returns an Array containing HTML-safe String instances. + # + # Example: + # texts_and_contexts + # => [{ text: '### Hello', + # context: { cache_key: [note, :note] } }] + def self.cache_collection_render(texts_and_contexts) + items_collection = texts_and_contexts.each_with_index do |item, index| + context = item[:context] + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + + item[:cache_key] = cache_key if cache_key + end + + cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + + items_in_cache = [] + items_not_in_cache = [] + + unless cacheable_items.empty? + items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) + items_not_in_cache = cacheable_items.reject do |item| + item[:rendered] = items_in_cache[item[:cache_key]] + items_in_cache.key?(item[:cache_key]) + end + end + + (items_not_in_cache + non_cacheable_items).each do |item| + item[:rendered] = render(item[:text], item[:context]) + Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] + end + + items_collection.map { |item| item[:rendered] } + end + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text @@ -78,5 +130,13 @@ module Banzai return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end + + # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. + # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key + # method. + def self.full_cache_multi_key(cache_key, pipeline_name) + return unless cache_key + Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) + end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 01ef13df57a..a48dc542b14 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -31,28 +31,34 @@ module Ci raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - builds.select do |build| - build[:stage] == stage && - process?(build[:only], build[:except], ref, tag, trigger_request) + def jobs_for_ref(ref, tag = false, trigger_request = nil) + @jobs.select do |_, job| + process?(job[:only], job[:except], ref, tag, trigger_request) end end - def builds - @jobs.map do |name, job| - build_job(name, job) + def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).select do |_, job| + job[:stage] == stage end end - def global_variables - @variables + def builds_for_ref(ref, tag = false, trigger_request = nil) + jobs_for_ref(ref, tag, trigger_request).map do |name, job| + build_job(name, job) + end end - def job_variables(name) - job = @jobs[name.to_sym] - return [] unless job + def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) + jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, job| + build_job(name, job) + end + end - job[:variables] || [] + def builds + @jobs.map do |name, job| + build_job(name, job) + end end private @@ -95,11 +101,10 @@ module Ci commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, - only: job[:only], - except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', environment: job[:environment], + yaml_variables: yaml_variables(name), options: { image: job[:image] || @image, services: job[:services] || @services, @@ -111,6 +116,24 @@ module Ci } end + def yaml_variables(name) + variables = global_variables.merge(job_variables(name)) + variables.map do |key, value| + { key: key, value: value, public: true } + end + end + + def global_variables + @variables || {} + end + + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job + + job[:variables] || {} + end + def validate! @jobs.each do |name, job| validate_job!(name, job) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 42232b7129d..2edddb84fc3 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,62 +7,91 @@ module ContainerRegistry MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + # Taken from: FaradayMiddleware::FollowRedirects + REDIRECT_CODES = Set.new [301, 302, 303, 307] + def initialize(base_uri, options = {}) @base_uri = base_uri - @faraday = Faraday.new(@base_uri) do |conn| - initialize_connection(conn, options) - end + @options = options end def repository_tags(name) - response_body @faraday.get("/v2/#{name}/tags/list") + response_body faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - response_body @faraday.get("/v2/#{name}/manifests/#{reference}") + response_body faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) - response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response = faraday.head("/v2/#{name}/manifests/#{reference}") response.headers['docker-content-digest'] if response.success? end def delete_repository_tag(name, reference) - @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + faraday.delete("/v2/#{name}/manifests/#{reference}").success? end def blob(name, digest, type = nil) - headers = {} - headers['Accept'] = type if type - response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) + type ||= 'application/octet-stream' + response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true end def delete_blob(name, digest) - @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + faraday.delete("/v2/#{name}/blobs/#{digest}").success? end - + private - + def initialize_connection(conn, options) conn.request :json + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + + def accept_manifest(conn) conn.headers['Accept'] = MANIFEST_VERSION conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + end - if options[:user] && options[:password] - conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) - elsif options[:token] - conn.request(:authorization, :bearer, options[:token].to_s) + def response_body(response, allow_redirect: false) + if allow_redirect && REDIRECT_CODES.include?(response.status) + response = redirect_response(response.headers['location']) end - conn.adapter :net_http + response.body if response && response.success? + end + + def redirect_response(location) + return unless location + + # We explicitly remove authorization token + faraday_blob.get(location) do |req| + req['Authorization'] = '' + end end - def response_body(response) - response.body if response.success? + def faraday + @faraday ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + accept_manifest(conn) + end + end + + def faraday_blob + @faraday_blob ||= Faraday.new(@base_uri) do |conn| + initialize_connection(conn, @options) + end end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 708d01b95a1..59040199920 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -53,7 +53,7 @@ module ContainerRegistry def config return unless config_blob - @config ||= ContainerRegistry::Config.new(self, config_blob) + @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data end def created_at diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 478f145bfed..ab94abeda77 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -63,7 +63,7 @@ module Grack def ci_request?(login, password) matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) - if project && matched_login.present? && git_cmd == 'git-upload-pack' + if project && matched_login.present? underscored_service = matched_login['s'].underscore if underscored_service == 'gitlab_ci' diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dec20d8659b..927f9dad20b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -20,11 +20,19 @@ module Gitlab if Database.postgresql? options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout end add_index(table_name, column_name, options) end + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. (PostgreSQL only) + def disable_statement_timeout + ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + end + # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. @@ -133,6 +141,8 @@ module Gitlab 'in the body of your migration class' end + disable_statement_timeout + transaction do add_column(table, column, type, default: nil) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 789c14518b0..28ad637fda4 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,16 +1,30 @@ module Gitlab module Diff class InlineDiff + # Regex to find a run of deleted lines followed by the same number of added lines + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze + attr_accessor :old_line, :new_line, :offset def self.for_lines(lines) - local_edit_indexes = self.find_local_edits(lines) + changed_line_pairs = self.find_changed_line_pairs(lines) inline_diffs = [] - local_edit_indexes.each do |index| - old_index = index - new_index = index + 1 + changed_line_pairs.each do |old_index, new_index| old_line = lines[old_index] new_line = lines[new_index] @@ -51,18 +65,28 @@ module Gitlab private - def self.find_local_edits(lines) - line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } - joined_line_prefixes = " #{line_prefixes.join} " - - offset = 0 - local_edit_indexes = [] - while index = joined_line_prefixes.index(" -+ ", offset) - local_edit_indexes << index - offset = index + 1 + # Finds pairs of old/new line pairs that represent the same line that changed + def self.find_changed_line_pairs(lines) + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + + changed_line_pairs = [] + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + changed_line_pairs << [i, i + changed_line_count] + end end - local_edit_indexes + changed_line_pairs end def longest_common_prefix(a, b) diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 1c1fc148123..b069afdd28c 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -8,95 +8,78 @@ module Gitlab end def parallelize - lines = [] - skip_next = false + i = 0 + free_right_index = nil + + lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - full_line = line.text - type = line.type line_code = diff_file.line_code(line) - line_new = line.new_pos - line_old = line.old_pos position = diff_file.position(line) - next_line = diff_file.next_line(line.index) - - if next_line - next_line = highlighted_diff_lines[next_line.index] - full_next_line = next_line.text - next_line_code = diff_file.line_code(next_line) - next_type = next_line.type - next_position = diff_file.position(next_line) - end - - case type + case line.type when 'match', nil # line in the right panel is the same as in the left one lines << { left: { - type: type, - number: line_old, - text: full_line, + type: line.type, + number: line.old_pos, + text: line.text, line_code: line_code, position: position }, right: { - type: type, - number: line_new, - text: full_line, + type: line.type, + number: line.new_pos, + text: line.text, line_code: line_code, position: position } } + + free_right_index = nil + i += 1 when 'old' - case next_type - when 'new' - # Left side has text removed, right side has text added - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - position: position - }, - right: { - type: next_type, - number: line_new, - text: full_next_line, - line_code: next_line_code, - position: next_position, - } - } - skip_next = true - when 'old', 'nonewline', nil - # Left side has text removed, right side doesn't have any change - # No next line code, no new line number, no new line text - lines << { - left: { - type: type, - number: line_old, - text: full_line, - line_code: line_code, - position: position - }, - right: { - type: next_type, - number: nil, - text: "", - line_code: nil, - position: nil - } + lines << { + left: { + type: line.type, + number: line.old_pos, + text: line.text, + line_code: line_code, + position: position + }, + right: { + type: nil, + number: nil, + text: "", + line_code: line_code, + position: position } - end + } + + # Once we come upon a new line it can be put on the right of this old line + free_right_index ||= i + i += 1 when 'new' - if skip_next - # Change has been already included in previous line so no need to do it again - skip_next = false - next + data = { + type: line.type, + number: line.new_pos, + text: line.text, + line_code: line_code, + position: position + } + + if free_right_index + # If an old line came before this without a line on the right, this + # line can be put to the right of it. + lines[free_right_index][:right] = data + + # If there are any other old lines on the left that don't yet have + # a new counterpart on the right, update the free_right_index + next_free_right_index = free_right_index + 1 + free_right_index = next_free_right_index < i ? next_free_right_index : nil else - # Change is only on the right side, left side has no change lines << { left: { type: nil, @@ -105,17 +88,15 @@ module Gitlab line_code: line_code, position: position }, - right: { - type: type, - number: line_new, - text: full_line, - line_code: line_code, - position: position - } + right: data } + + free_right_index = nil + i += 1 end end end + lines end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 043f10d96a9..084e514492c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -78,10 +78,21 @@ module Gitlab def rate_limit api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? end def rate_limit_exceed? - rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS end def rate_limit_sleep_time diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 3f76ec97977..46d40f75be6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -15,31 +15,35 @@ module Gitlab end def execute - project_identifier = CGI.escape(project.import_source) - - # Issues && Comments - issues = client.issues(project_identifier) - - issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"]) - body += issue["description"] - - comments = client.issue_comments(project_identifier, issue["id"]) - - if comments.any? - body += @formatter.comments_header + ActiveRecord::Base.no_touching do + project_identifier = CGI.escape(project.import_source) + + # Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] + + comments = client.issue_comments(project_identifier, issue["id"]) + + if comments.any? + body += @formatter.comments_header + end + + comments.each do |comment| + body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + end + + project.issues.create!( + iid: issue["iid"], + description: body, + title: issue["title"], + state: issue["state"], + updated_at: issue["updated_at"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) end - - comments.each do |comment| - body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) - end - - project.issues.create!( - description: body, - title: issue["title"], - state: issue["state"], - author_id: gl_user_id(project, issue["author"]["id"]) - ) end true diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 41296415e35..9360afedfcb 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap, repository: repository). + def self.highlight(blob_name, blob_content, repository: nil, plain: false) + new(blob_name, blob_content, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -13,30 +13,34 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - attr_reader :lexer - def initialize(blob_name, blob_content, repository: nil, nowrap: true) + def initialize(blob_name, blob_content, repository: nil) + @formatter = Rouge::Formatters::HTMLGitlab.new + @repository = repository @blob_name = blob_name @blob_content = blob_content - @repository = repository - @formatter = rouge_formatter(nowrap: nowrap) - - @lexer = custom_language || begin - Rouge::Lexer.guess(filename: blob_name, source: blob_content).new - rescue Rouge::Lexer::AmbiguousGuess => e - e.alternatives.sort_by(&:tag).first - end end def highlight(text, continue: true, plain: false) if plain - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + hl_lexer = Rouge::Lexers::PlainText + continue = false else - @formatter.format(@lexer.lex(text, continue: continue)).html_safe + hl_lexer = self.lexer end + + @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe rescue @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end + def lexer + @lexer ||= custom_language || begin + Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.sort_by(&:tag).first + end + end + private def custom_language @@ -46,16 +50,5 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end - - def rouge_formatter(options = {}) - options = options.reverse_merge( - nowrap: true, - cssclass: 'code highlight', - lineanchors: true, - lineanchorsid: 'LC' - ) - - Rouge::Formatters::HTMLGitlab.new(options) - end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 588647e5adb..bab2ea73c4f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,6 +3,7 @@ module Gitlab extend self VERSION = '0.1.1' + FILENAME_LIMIT = 50 def export_path(relative_path:) File.join(storage_path, relative_path) @@ -28,6 +29,12 @@ module Gitlab 'VERSION' end + def export_filename(project:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}" + + "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" + end + def version VERSION end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 8f66f48cbfe..6b69a653f12 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -44,8 +44,7 @@ module Gitlab def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project), - wiki: true) + project: ProjectWiki.new(project_tree.restored_project)) end def uploads_restorer diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 025ecc12f9f..051110c23cf 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -12,7 +12,10 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') - create_relations + + ActiveRecord::Base.no_touching do + create_relations + end rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 9824df3f274..6ba25a31641 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -87,7 +87,7 @@ module Gitlab project_id = @relation_hash.delete('project_id') # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['project_id'] = project_id @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] @@ -111,7 +111,7 @@ module Gitlab end def imported_object - imported_object = relation_class.new(@relation_hash) + imported_object = relation_class.new(parsed_relation_hash) yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object @@ -125,6 +125,10 @@ module Gitlab def admin_user? @user.is_admin? end + + def parsed_relation_hash + @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 546dae4d122..f84de652a57 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -3,15 +3,14 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:, wiki: false) + def initialize(project:, shared:, path_to_bundle:) @project = project @path_to_bundle = path_to_bundle @shared = shared - @wiki = wiki end def restore - return wiki? unless File.exist?(@path_to_bundle) + return true unless File.exist?(@path_to_bundle) FileUtils.mkdir_p(path_to_repo) @@ -30,10 +29,6 @@ module Gitlab def path_to_repo @project.repository.path_to_repo end - - def wiki? - @wiki - end end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index cce43fe994b..331e14021e6 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -11,7 +11,7 @@ module Gitlab end def save - return false if @project.empty_repo? + return true if @project.empty_repo? # it's ok to have no repo @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) bundle_to_disk diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 6a60b65071f..6130c124dd1 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -7,7 +7,8 @@ module Gitlab new(*args).save end - def initialize(shared:) + def initialize(project:, shared:) + @project = project @shared = shared end @@ -36,7 +37,7 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + @archive_file ||= File.join(@shared.export_path, '..', Gitlab::ImportExport.export_filename(project: @project)) end end end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 1eedae39f8a..6107420e4dd 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab def save @wiki = ProjectWiki.new(@project) return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) end diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index 811363405a8..a1ee1aa81ff 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -47,6 +47,8 @@ module Gitlab end def render_storage_upload_store_response(oid, size, tmp_file_name) + return render_forbidden unless tmp_file_name + render_response_to_push do render_lfs_upload_ok(oid, size, tmp_file_name) end diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb index 69bd5e62305..f2a76a56b8f 100644 --- a/lib/gitlab/lfs/router.rb +++ b/lib/gitlab/lfs/router.rb @@ -74,8 +74,6 @@ module Gitlab lfs.render_storage_upload_authorize_response(oid, size) else tmp_file_name = sanitize_tmp_filename(@request.env['HTTP_X_GITLAB_LFS_TMP']) - return nil unless tmp_file_name - lfs.render_storage_upload_store_response(oid, size, tmp_file_name) end end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 3358ed6773e..f818dc78d34 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -1,171 +1,27 @@ -require 'cgi' - module Rouge module Formatters - class HTMLGitlab < Rouge::Formatter + class HTMLGitlab < Rouge::Formatters::HTML tag 'html_gitlab' # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # - # [+nowrap+] If set to True, don't wrap the output at all, not - # even inside a <tt><pre></tt> tag (default: false). - # [+cssclass+] CSS class for the wrapping <tt><div></tt> tag - # (default: 'highlight'). - # [+linenos+] If set to 'table', output line numbers as a table - # with two cells, one containing the line numbers, - # the other the whole code. This is copy paste friendly, - # but may cause alignment problems with some browsers - # or fonts. If set to 'inline', the line numbers will - # be integrated in the <tt><pre></tt> tag that contains - # the code (default: nil). # [+linenostart+] The line number for the first line (default: 1). - # [+lineanchors+] If set to true the formatter will wrap each output - # line in an anchor tag with a name of L-linenumber. - # This allows easy linking to certain lines - # (default: false). - # [+lineanchorsid+] If lineanchors is true the name of the anchors can - # be changed with lineanchorsid to e.g. foo-linenumber - # (default: 'L'). - # [+anchorlinenos+] If set to true, will wrap line numbers in <tt><a></tt> - # tags. Used in combination with linenos and lineanchors - # (default: false). - # [+inline_theme+] Inline CSS styles for the <pre> tag (default: false). - def initialize( - nowrap: false, - cssclass: 'highlight', - linenos: nil, - linenostart: 1, - lineanchors: false, - lineanchorsid: 'L', - anchorlinenos: false, - inline_theme: nil - ) - @nowrap = nowrap - @cssclass = cssclass - @linenos = linenos + def initialize(linenostart: 1) @linenostart = linenostart - @lineanchors = lineanchors - @lineanchorsid = lineanchorsid - @anchorlinenos = anchorlinenos - @inline_theme = Theme.find(inline_theme).new if inline_theme.is_a?(String) - end - - def render(tokens) - case @linenos - when 'table' - render_tableized(tokens) - when 'inline' - render_untableized(tokens) - else - render_untableized(tokens) - end - end - - alias_method :format, :render - - private - - def render_untableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap - html << wrap_lines(data[:code]) - html << "</code></pre>\n" unless @nowrap - html + @line_number = linenostart end - def render_tableized(tokens) - data = process_tokens(tokens) - - html = '' - html << "<div class=\"#{@cssclass}\">" unless @nowrap - html << '<table><tbody>' - html << "<td class=\"linenos\"><pre>" - html << wrap_linenos(data[:numbers]) - html << '</pre></td>' - html << "<td class=\"lines\"><pre><code>" - html << wrap_lines(data[:code]) - html << '</code></pre></td>' - html << '</tbody></table>' - html << '</div>' unless @nowrap - html - end - - def process_tokens(tokens) - rendered = [] - current_line = '' - - tokens.each do |tok, val| - # In the case of multi-line values (e.g. comments), we need to apply - # styling to each line since span elements are inline. - val.lines.each do |line| - stripped = line.chomp - current_line << span(tok, stripped) - - if line.end_with?("\n") - rendered << current_line - current_line = '' - end - end - end - - # Add leftover text - rendered << current_line if current_line.present? - - num_lines = rendered.size - numbers = (@linenostart..num_lines + @linenostart - 1).to_a - - { numbers: numbers, code: rendered } - end - - def wrap_linenos(numbers) - if @anchorlinenos - numbers.map! do |number| - "<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>" - end - end - numbers.join("\n") - end - - def wrap_lines(lines) - if @lineanchors - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - - if @linenos == 'inline' - "<a name=\"L#{number}\"></a>" \ - "<span class=\"linenos\">#{number}</span>" \ - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - else - "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \ - '</span>' - end - end - elsif @linenos == 'inline' - lines = lines.each_with_index.map do |line, index| - number = index + @linenostart - "<span class=\"linenos\">#{number}</span>#{line}" - end - end - - lines.join("\n") - end + def stream(tokens, &b) + is_first = true + token_lines(tokens) do |line| + yield "\n" unless is_first + is_first = false - def span(tok, val) - # http://stackoverflow.com/a/1600584/2587286 - val = CGI.escapeHTML(val) + yield %(<span id="LC#{@line_number}" class="line">) + line.each { |token, value| yield span(token, value) } + yield %(</span>) - if tok.shortname.empty? - val - else - if @inline_theme - rules = @inline_theme.style_for(tok).rendered_rules - "<span style=\"#{rules.to_a.join(';')}\"#{val}</span>" - else - "<span class=\"#{tok.shortname}\">#{val}</span>" - end + @line_number += 1 end end end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..4a4892a2e07 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -49,7 +49,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..0b93d7f292f 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -93,7 +93,12 @@ server { proxy_http_version 1.1; - proxy_set_header Host $http_host; + ## By overwriting Host and clearing X-Forwarded-Host we ensure that + ## internal HTTP redirects generated by GitLab always send users to + ## YOUR_SERVER_FQDN. + proxy_set_header Host YOUR_SERVER_FQDN; + proxy_set_header X-Forwarded-Host ""; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |