summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--app/models/merge_request_diff.rb2
-rw-r--r--app/validators/dynamic_path_validator.rb2
-rw-r--r--lib/gitlab/encoding_helper.rb62
-rw-r--r--lib/gitlab/git/blame.rb2
-rw-r--r--lib/gitlab/git/blob.rb2
-rw-r--r--lib/gitlab/git/commit.rb2
-rw-r--r--lib/gitlab/git/diff.rb2
-rw-r--r--lib/gitlab/git/encoding_helper.rb64
-rw-r--r--lib/gitlab/git/ref.rb2
-rw-r--r--lib/gitlab/git/tree.rb2
-rw-r--r--spec/lib/gitlab/encoding_helper_spec.rb (renamed from spec/lib/gitlab/git/encoding_helper_spec.rb)4
-rw-r--r--spec/lib/gitlab/git/repository_spec.rb2
12 files changed, 73 insertions, 75 deletions
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 1c2f335f95e..99dd2130188 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -1,7 +1,7 @@
class MergeRequestDiff < ActiveRecord::Base
include Sortable
include Importable
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
# Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE = 100
diff --git a/app/validators/dynamic_path_validator.rb b/app/validators/dynamic_path_validator.rb
index a9b76c7c960..27ac60637fd 100644
--- a/app/validators/dynamic_path_validator.rb
+++ b/app/validators/dynamic_path_validator.rb
@@ -6,7 +6,7 @@
# Values are checked for formatting and exclusion from a list of illegal path
# names.
class DynamicPathValidator < ActiveModel::EachValidator
- extend Gitlab::Git::EncodingHelper
+ extend Gitlab::EncodingHelper
class << self
def valid_user_path?(path)
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
new file mode 100644
index 00000000000..dbe28e6bb93
--- /dev/null
+++ b/lib/gitlab/encoding_helper.rb
@@ -0,0 +1,62 @@
+module Gitlab
+ module EncodingHelper
+ extend self
+
+ # This threshold is carefully tweaked to prevent usage of encodings detected
+ # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
+ # we're better off sticking with utf8 encoding.
+ # Reason: git diff can return strings with invalid utf8 byte sequences if it
+ # truncates a diff in the middle of a multibyte character. In this case
+ # CharlockHolmes will try to guess the encoding and will likely suggest an
+ # obscure encoding with low confidence.
+ # There is a lot more info with this merge request:
+ # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
+ ENCODING_CONFIDENCE_THRESHOLD = 40
+
+ def encode!(message)
+ return nil unless message.respond_to? :force_encoding
+
+ # if message is utf-8 encoding, just return it
+ message.force_encoding("UTF-8")
+ return message if message.valid_encoding?
+
+ # return message if message type is binary
+ detect = CharlockHolmes::EncodingDetector.detect(message)
+ return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+ # force detected encoding if we have sufficient confidence.
+ if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+ message.force_encoding(detect[:encoding])
+ end
+
+ # encode and clean the bad chars
+ message.replace clean(message)
+ rescue
+ encoding = detect ? detect[:encoding] : "unknown"
+ "--broken encoding: #{encoding}"
+ end
+
+ def encode_utf8(message)
+ detect = CharlockHolmes::EncodingDetector.detect(message)
+ if detect
+ begin
+ CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+ rescue ArgumentError => e
+ Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
+
+ ''
+ end
+ else
+ clean(message)
+ end
+ end
+
+ private
+
+ def clean(message)
+ message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+ .encode("UTF-8")
+ .gsub("\0".encode("UTF-8"), "")
+ end
+ end
+end
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 58193391926..66829a03c2e 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,7 +1,7 @@
module Gitlab
module Git
class Blame
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
attr_reader :lines, :blames
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 6b0a66365a7..d60e607b02b 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -2,7 +2,7 @@ module Gitlab
module Git
class Blob
include Linguist::BlobHelper
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
# This number is the maximum amount of data that we want to display to
# the user. We load as much as we can for encoding detection
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 297531db4cc..bb04731f08c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -2,7 +2,7 @@
module Gitlab
module Git
class Commit
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
attr_accessor :raw_commit, :head, :refs
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index ccccca96595..0594ac8e213 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -3,7 +3,7 @@ module Gitlab
module Git
class Diff
TimeoutError = Class.new(StandardError)
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
# Diff properties
attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff
diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb
deleted file mode 100644
index f918074cb14..00000000000
--- a/lib/gitlab/git/encoding_helper.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-module Gitlab
- module Git
- module EncodingHelper
- extend self
-
- # This threshold is carefully tweaked to prevent usage of encodings detected
- # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
- # we're better off sticking with utf8 encoding.
- # Reason: git diff can return strings with invalid utf8 byte sequences if it
- # truncates a diff in the middle of a multibyte character. In this case
- # CharlockHolmes will try to guess the encoding and will likely suggest an
- # obscure encoding with low confidence.
- # There is a lot more info with this merge request:
- # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
- ENCODING_CONFIDENCE_THRESHOLD = 40
-
- def encode!(message)
- return nil unless message.respond_to? :force_encoding
-
- # if message is utf-8 encoding, just return it
- message.force_encoding("UTF-8")
- return message if message.valid_encoding?
-
- # return message if message type is binary
- detect = CharlockHolmes::EncodingDetector.detect(message)
- return message.force_encoding("BINARY") if detect && detect[:type] == :binary
-
- # force detected encoding if we have sufficient confidence.
- if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
- message.force_encoding(detect[:encoding])
- end
-
- # encode and clean the bad chars
- message.replace clean(message)
- rescue
- encoding = detect ? detect[:encoding] : "unknown"
- "--broken encoding: #{encoding}"
- end
-
- def encode_utf8(message)
- detect = CharlockHolmes::EncodingDetector.detect(message)
- if detect
- begin
- CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
- rescue ArgumentError => e
- Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
-
- ''
- end
- else
- clean(message)
- end
- end
-
- private
-
- def clean(message)
- message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
- .encode("UTF-8")
- .gsub("\0".encode("UTF-8"), "")
- end
- end
- end
-end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index 37ef6836742..ebf7393dc61 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,7 +1,7 @@
module Gitlab
module Git
class Ref
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
# Branch or tag name
# without "refs/tags|heads" prefix
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index d41256d9a84..b9afa05c819 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,7 +1,7 @@
module Gitlab
module Git
class Tree
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
attr_accessor :id, :root_id, :name, :path, :type,
:mode, :commit_id, :submodule_url
diff --git a/spec/lib/gitlab/git/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb
index 48fc817d857..1482ef7132d 100644
--- a/spec/lib/gitlab/git/encoding_helper_spec.rb
+++ b/spec/lib/gitlab/encoding_helper_spec.rb
@@ -1,7 +1,7 @@
require "spec_helper"
-describe Gitlab::Git::EncodingHelper do
- let(:ext_class) { Class.new { extend Gitlab::Git::EncodingHelper } }
+describe Gitlab::EncodingHelper do
+ let(:ext_class) { Class.new { extend Gitlab::EncodingHelper } }
let(:binary_string) { File.read(Rails.root + "spec/fixtures/dk.png") }
describe '#encode!' do
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index 9d0e95d5b19..26215381cc4 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -1,7 +1,7 @@
require "spec_helper"
describe Gitlab::Git::Repository, seed_helper: true do
- include Gitlab::Git::EncodingHelper
+ include Gitlab::EncodingHelper
let(:repository) { Gitlab::Git::Repository.new('default', TEST_REPO_PATH) }
View Lorry Controller Status