Merge branch 'master' into decouple-member-notification

17 files changed, 267 insertions, 25 deletions

diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb
new file mode 100644
index 00000000000..e5e9fab3f5c
--- /dev/null
+++ b/lib/gitlab/badge/build.rb
@@ -0,0 +1,46 @@
+module Gitlab
+  module Badge
+    ##
+    # Build badge
+    #
+    class Build
+      include Gitlab::Application.routes.url_helpers
+      include ActionView::Helpers::AssetTagHelper
+      include ActionView::Helpers::UrlHelper
+
+      def initialize(project, ref)
+        @project, @ref = project, ref
+        @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref)
+      end
+
+      def type
+        'image/svg+xml'
+      end
+
+      def data
+        File.read(@image[:path])
+      end
+
+      def to_s
+        @image[:name].sub(/\.svg$/, '')
+      end
+
+      def to_html
+        link_to(image_tag(image_url, alt: 'build status'), link_url)
+      end
+
+      def to_markdown
+        "[![build status](#{image_url})](#{link_url})"
+      end
+
+      def image_url
+        build_namespace_project_badges_url(@project.namespace,
+                                           @project, @ref, format: :svg)
+      end
+
+      def link_url
+        namespace_project_commits_url(@project.namespace, @project, id: @ref)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 761b63e98f6..1acc22fe5bf 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -21,7 +21,6 @@ module Gitlab
         default_branch_protection: Settings.gitlab['default_branch_protection'],
         signup_enabled: Settings.gitlab['signup_enabled'],
         signin_enabled: Settings.gitlab['signin_enabled'],
-        twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'],
         gravatar_enabled: Settings.gravatar['enabled'],
         sign_in_text: Settings.extra['sign_in_text'],
         restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 41f0edcaf7e..8f9be6cd9a3 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -5,7 +5,7 @@ module Gitlab
         attr_accessor :recipient
         attr_reader :author_id, :ref, :action
 
-        include Gitlab::Application.routes.url_helpers
+        include Gitlab::Routing.url_helpers
 
         delegate :namespace, :name_with_namespace, to: :project, prefix: :project
         delegate :name, to: :author, prefix: :author
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index d4b6f6d120d..97ef9851d71 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -63,6 +63,10 @@ module Gitlab
       end
 
       def reply_key
+        key_from_to_header || key_from_additional_headers
+      end
+
+      def key_from_to_header
         key = nil
         message.to.each do |address|
           key = Gitlab::IncomingEmail.key_from_address(address)
@@ -72,6 +76,17 @@ module Gitlab
         key
       end
 
+      def key_from_additional_headers
+        reply_key = nil
+
+        Array(message.references).each do |message_id|
+          reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id)
+          break if reply_key
+        end
+
+        reply_key
+      end
+
       def sent_notification
         return nil unless reply_key
 
diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb
index 431d50882fd..2152182b37f 100644
--- a/lib/gitlab/fogbugz_import/client.rb
+++ b/lib/gitlab/fogbugz_import/client.rb
@@ -26,7 +26,7 @@ module Gitlab
       def user_map
         users = {}
         res = @api.command(:listPeople)
-        res['people']['person'].each do |user|
+        [res['people']['person']].flatten.each do |user|
           users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] }
         end
         users
diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb
index a1c6ee7bd69..78d7a4f27cf 100644
--- a/lib/gitlab/gfm/reference_rewriter.rb
+++ b/lib/gitlab/gfm/reference_rewriter.rb
@@ -34,16 +34,21 @@ module Gitlab
         @source_project = source_project
         @current_user = current_user
         @original_html = markdown(text)
+        @pattern = Gitlab::ReferenceExtractor.references_pattern
       end
 
       def rewrite(target_project)
-        pattern = Gitlab::ReferenceExtractor.references_pattern
+        return @text unless needs_rewrite?
 
-        @text.gsub(pattern) do |reference|
+        @text.gsub(@pattern) do |reference|
           unfold_reference(reference, Regexp.last_match, target_project)
         end
       end
 
+      def needs_rewrite?
+        @text =~ @pattern
+      end
+
       private
 
       def unfold_reference(reference, match, target_project)
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
new file mode 100644
index 00000000000..abc8c8c55e6
--- /dev/null
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -0,0 +1,51 @@
+module Gitlab
+  module Gfm
+    ##
+    # Class that rewrites markdown links for uploads
+    #
+    # Using a pattern defined in `FileUploader` it copies files to a new
+    # project and rewrites all links to uploads in in a given text.
+    #
+    #
+    class UploadsRewriter
+      def initialize(text, source_project, _current_user)
+        @text = text
+        @source_project = source_project
+        @pattern = FileUploader::MARKDOWN_PATTERN
+      end
+
+      def rewrite(target_project)
+        return @text unless needs_rewrite?
+
+        @text.gsub(@pattern) do |markdown|
+          file = find_file(@source_project, $~[:secret], $~[:file])
+          return markdown unless file.try(:exists?)
+
+          new_uploader = FileUploader.new(target_project)
+          new_uploader.store!(file)
+          new_uploader.to_markdown
+        end
+      end
+
+      def needs_rewrite?
+        files.any?
+      end
+
+      def files
+        referenced_files = @text.scan(@pattern).map do
+          find_file(@source_project, $~[:secret], $~[:file])
+        end
+
+        referenced_files.compact.select(&:exists?)
+      end
+
+      private
+
+      def find_file(project, secret, file)
+        uploader = FileUploader.new(project, secret)
+        uploader.retrieve_from_store!(file)
+        uploader.file
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index 9068d79c95e..8ce9d32abe0 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -1,13 +1,10 @@
 module Gitlab
   module IncomingEmail
     class << self
-      def enabled?
-        config.enabled && address_formatted_correctly?
-      end
+      FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
 
-      def address_formatted_correctly?
-        config.address &&
-          config.address.include?("%{key}")
+      def enabled?
+        config.enabled && config.address
       end
 
       def reply_address(key)
@@ -24,6 +21,13 @@ module Gitlab
         match[1]
       end
 
+      def key_from_fallback_reply_message_id(message_id)
+        match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX)
+        return unless match
+
+        match[1]
+      end
+
       def config
         Gitlab.config.incoming_email
       end
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index da4435c7308..f2b649e50a2 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -33,7 +33,10 @@ module Gitlab
 
       def allowed?
         if ldap_user
-          return true unless ldap_config.active_directory
+          unless ldap_config.active_directory
+            user.activate if user.ldap_blocked?
+            return true
+          end
 
           # Block user in GitLab if he/she was blocked in AD
           if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter)
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 88a265c6af2..4a3f47b5a95 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -70,6 +70,32 @@ module Gitlab
       value.to_s.gsub('=', '\\=')
     end
 
+    # Measures the execution time of a block.
+    #
+    # Example:
+    #
+    #     Gitlab::Metrics.measure(:find_by_username_timings) do
+    #       User.find_by_username(some_username)
+    #     end
+    #
+    # series - The name of the series to store the data in.
+    # values - A Hash containing extra values to add to the metric.
+    # tags - A Hash containing extra tags to add to the metric.
+    #
+    # Returns the value yielded by the supplied block.
+    def self.measure(series, values = {}, tags = {})
+      return yield unless Transaction.current
+
+      start = Time.now.to_f
+      retval = yield
+      duration = (Time.now.to_f - start) * 1000.0
+      values = values.merge(duration: duration)
+
+      Transaction.current.add_metric(series, values, tags)
+
+      retval
+    end
+
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index 7ea9555cc8c..1cd1ca30f70 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -2,6 +2,8 @@ module Gitlab
   module Metrics
     # Class for storing details of a single metric (label, value, etc).
     class Metric
+      JITTER_RANGE = 0.000001..0.001
+
       attr_reader :series, :values, :tags, :created_at
 
       # series - The name of the series (as a String) to store the metric in.
@@ -16,11 +18,29 @@ module Gitlab
 
       # Returns a Hash in a format that can be directly written to InfluxDB.
       def to_hash
+        # InfluxDB overwrites an existing point if a new point has the same
+        # series, tag set, and timestamp. In a highly concurrent environment
+        # this means that using the number of seconds since the Unix epoch is
+        # inevitably going to collide with another timestamp. For example, two
+        # Rails requests processed by different processes may end up generating
+        # metrics using the _exact_ same timestamp (in seconds).
+        #
+        # Due to the way InfluxDB is set up there's no solution to this problem,
+        # all we can do is lower the amount of collisions. We do this by using
+        # Time#to_f which returns the seconds as a Float providing greater
+        # accuracy. We then add a small random value that is large enough to
+        # distinguish most timestamps but small enough to not alter the amount
+        # of seconds.
+        #
+        # See https://gitlab.com/gitlab-com/operations/issues/175 for more
+        # information.
+        time = @created_at.to_f + rand(JITTER_RANGE)
+
         {
           series:    @series,
           tags:      @tags,
           values:    @values,
-          timestamp: @created_at.to_i * 1_000_000_000
+          timestamp: (time * 1_000_000_000).to_i
         }
       end
     end
diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb
index 71cf6a0d886..18523e0aefe 100644
--- a/lib/gitlab/note_data_builder.rb
+++ b/lib/gitlab/note_data_builder.rb
@@ -41,7 +41,7 @@ module Gitlab
           data[:issue] = note.noteable.hook_attrs
         elsif note.for_merge_request?
           data[:merge_request] = note.noteable.hook_attrs
-        elsif note.for_project_snippet?
+        elsif note.for_snippet?
           data[:snippet] = note.noteable.hook_attrs
         end
 
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
new file mode 100644
index 00000000000..5132177de51
--- /dev/null
+++ b/lib/gitlab/routing.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Routing
+    # Returns the URL helpers Module.
+    #
+    # This method caches the output as Rails' "url_helpers" method creates an
+    # anonymous module every time it's called.
+    #
+    # Returns a Module.
+    def self.url_helpers
+      @url_helpers ||= Gitlab::Application.routes.url_helpers
+    end
+  end
+end
diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb
new file mode 100644
index 00000000000..32c1c9ec5bb
--- /dev/null
+++ b/lib/gitlab/saml/auth_hash.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Saml
+    class AuthHash < Gitlab::OAuth::AuthHash
+
+      def groups
+        get_raw(Gitlab::Saml::Config.groups)
+      end
+
+      private
+
+      def get_raw(key)
+        # Needs to call `all` because of https://git.io/vVo4u
+        # otherwise just the first value is returned
+        auth_hash.extra[:raw_info].all[key]
+      end
+
+    end
+  end
+end
diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb
new file mode 100644
index 00000000000..0f40c00f547
--- /dev/null
+++ b/lib/gitlab/saml/config.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Saml
+    class Config
+
+      class << self
+        def options
+          Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' }
+        end
+
+        def groups
+          options[:groups_attribute]
+        end
+
+        def external_groups
+          options[:external_groups]
+        end
+      end
+
+    end
+  end
+end
diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb
index b1e30110ef5..c1072452abe 100644
--- a/lib/gitlab/saml/user.rb
+++ b/lib/gitlab/saml/user.rb
@@ -18,7 +18,7 @@ module Gitlab
           @user ||= find_or_create_ldap_user
         end
 
-        if auto_link_saml_enabled?
+        if auto_link_saml_user?
           @user ||= find_by_email
         end
 
@@ -26,6 +26,16 @@ module Gitlab
           @user ||= build_new_user
         end
 
+        if external_users_enabled?
+          # Check if there is overlap between the user's groups and the external groups
+          # setting then set user as external or internal.
+          if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty?
+            @user.external = false
+          else
+            @user.external = true
+          end
+        end
+
         @user
       end
 
@@ -37,11 +47,23 @@ module Gitlab
         end
       end
 
+      def changed?
+        gl_user.changed? || gl_user.identities.any?(&:changed?)
+      end
+
       protected
 
-      def auto_link_saml_enabled?
+      def auto_link_saml_user?
         Gitlab.config.omniauth.auto_link_saml_user
       end
+
+      def external_users_enabled?
+        !Gitlab::Saml::Config.external_groups.nil?
+      end
+
+      def auth_hash=(auth_hash)
+        @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash)
+      end
     end
   end
 end
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 6f0d02cafd1..f301d42939d 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -1,7 +1,8 @@
 module Gitlab
   class UrlBuilder
-    include Gitlab::Application.routes.url_helpers
+    include Gitlab::Routing.url_helpers
     include GitlabRoutingHelper
+    include ActionView::RecordIdentifier
 
     def initialize(type)
       @type = type
@@ -37,19 +38,16 @@ module Gitlab
         namespace_project_commit_url(namespace_id: note.project.namespace,
                                      id: note.commit_id,
                                      project_id: note.project,
-                                     anchor: "note_#{note.id}")
+                                     anchor: dom_id(note))
       elsif note.for_issue?
         issue = Issue.find(note.noteable_id)
-        issue_url(issue,
-                  anchor: "note_#{note.id}")
+        issue_url(issue, anchor: dom_id(note))
       elsif note.for_merge_request?
         merge_request = MergeRequest.find(note.noteable_id)
-        merge_request_url(merge_request,
-                          anchor: "note_#{note.id}")
-      elsif note.for_project_snippet?
+        merge_request_url(merge_request, anchor: dom_id(note))
+      elsif note.for_snippet?
         snippet = Snippet.find(note.noteable_id)
-        project_snippet_url(snippet,
-                            anchor: "note_#{note.id}")
+        project_snippet_url(snippet, anchor: dom_id(note))
       end
     end
   end