Merge branch 'stanhu/gitlab-ce-add-eager-load-lib' into 'master'

Add eager load paths to help prevent dependency load issues with Sidekiq workers

_Originally opened at !3545 by @stanhu._

- - -

Relevant resources:

- https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
- https://github.com/mperham/sidekiq/issues/1281#issuecomment-27129904
- http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
- https://github.com/rails/rails/blob/52ce6ece8c8f74064bb64e0a0b1ddd83092718e1/railties/lib/rails/engine.rb#L472-L479
- https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/paths.rb

Attempts to address #3661, #11896, #12769, #13521, #14131, #14589, #14759, #14825.

See merge request !3724

5 files changed, 177 insertions, 176 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index cc1004f8005..5fd9c30cb42 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -1,5 +1,3 @@
-Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file}
-
 module API
   class API < Grape::API
     include APIGuard
@@ -25,38 +23,39 @@ module API
     format :json
     content_type :txt, "text/plain"
 
-    helpers Helpers
-
-    mount Groups
-    mount GroupMembers
-    mount Users
-    mount Projects
-    mount Repositories
-    mount Issues
-    mount Milestones
-    mount Session
-    mount MergeRequests
-    mount Notes
-    mount Internal
-    mount SystemHooks
-    mount ProjectSnippets
-    mount ProjectMembers
-    mount DeployKeys
-    mount ProjectHooks
-    mount Services
-    mount Files
-    mount Commits
-    mount CommitStatus
-    mount Namespaces
-    mount Branches
-    mount Labels
-    mount Settings
-    mount Keys
-    mount Tags
-    mount Triggers
-    mount Builds
-    mount Variables
-    mount Runners
-    mount Licenses
+    # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
+    helpers ::API::Helpers
+
+    mount ::API::Groups
+    mount ::API::GroupMembers
+    mount ::API::Users
+    mount ::API::Projects
+    mount ::API::Repositories
+    mount ::API::Issues
+    mount ::API::Milestones
+    mount ::API::Session
+    mount ::API::MergeRequests
+    mount ::API::Notes
+    mount ::API::Internal
+    mount ::API::SystemHooks
+    mount ::API::ProjectSnippets
+    mount ::API::ProjectMembers
+    mount ::API::DeployKeys
+    mount ::API::ProjectHooks
+    mount ::API::Services
+    mount ::API::Files
+    mount ::API::Commits
+    mount ::API::CommitStatuses
+    mount ::API::Namespaces
+    mount ::API::Branches
+    mount ::API::Labels
+    mount ::API::Settings
+    mount ::API::Keys
+    mount ::API::Tags
+    mount ::API::Triggers
+    mount ::API::Builds
+    mount ::API::Variables
+    mount ::API::Runners
+    mount ::API::Licenses
   end
 end
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index b9994fcefda..7e67edb203a 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -2,171 +2,175 @@
 
 require 'rack/oauth2'
 
-module APIGuard
-  extend ActiveSupport::Concern
+module API
+  module APIGuard
+    extend ActiveSupport::Concern
 
-  included do |base|
-    # OAuth2 Resource Server Authentication
-    use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
-      # The authenticator only fetches the raw token string
+    included do |base|
+      # OAuth2 Resource Server Authentication
+      use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
+        # The authenticator only fetches the raw token string
 
-      # Must yield access token to store it in the env
-      request.access_token
-    end
+        # Must yield access token to store it in the env
+        request.access_token
+      end
 
-    helpers HelperMethods
+      helpers HelperMethods
 
-    install_error_responders(base)
-  end
+      install_error_responders(base)
+    end
 
-  # Helper Methods for Grape Endpoint
-  module HelperMethods
-    # Invokes the doorkeeper guard.
-    #
-    # If token is presented and valid, then it sets @current_user.
-    #
-    # If the token does not have sufficient scopes to cover the requred scopes,
-    # then it raises InsufficientScopeError.
-    #
-    # If the token is expired, then it raises ExpiredError.
-    #
-    # If the token is revoked, then it raises RevokedError.
-    #
-    # If the token is not found (nil), then it raises TokenNotFoundError.
-    #
-    # Arguments:
-    #
-    #   scopes: (optional) scopes required for this guard.
-    #           Defaults to empty array.
-    #
-    def doorkeeper_guard!(scopes: [])
-      if (access_token = find_access_token).nil?
-        raise TokenNotFoundError
-
-      else
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
-          raise InsufficientScopeError.new(scopes)
-        when Oauth2::AccessTokenValidationService::EXPIRED
-          raise ExpiredError
-        when Oauth2::AccessTokenValidationService::REVOKED
-          raise RevokedError
-        when Oauth2::AccessTokenValidationService::VALID
-          @current_user = User.find(access_token.resource_owner_id)
+    # Helper Methods for Grape Endpoint
+    module HelperMethods
+      # Invokes the doorkeeper guard.
+      #
+      # If token is presented and valid, then it sets @current_user.
+      #
+      # If the token does not have sufficient scopes to cover the requred scopes,
+      # then it raises InsufficientScopeError.
+      #
+      # If the token is expired, then it raises ExpiredError.
+      #
+      # If the token is revoked, then it raises RevokedError.
+      #
+      # If the token is not found (nil), then it raises TokenNotFoundError.
+      #
+      # Arguments:
+      #
+      #   scopes: (optional) scopes required for this guard.
+      #           Defaults to empty array.
+      #
+      def doorkeeper_guard!(scopes: [])
+        if (access_token = find_access_token).nil?
+          raise TokenNotFoundError
+
+        else
+          case validate_access_token(access_token, scopes)
+          when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+            raise InsufficientScopeError.new(scopes)
+          when Oauth2::AccessTokenValidationService::EXPIRED
+            raise ExpiredError
+          when Oauth2::AccessTokenValidationService::REVOKED
+            raise RevokedError
+          when Oauth2::AccessTokenValidationService::VALID
+            @current_user = User.find(access_token.resource_owner_id)
+          end
         end
       end
-    end
 
-    def doorkeeper_guard(scopes: [])
-      if access_token = find_access_token
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
-          raise InsufficientScopeError.new(scopes)
+      def doorkeeper_guard(scopes: [])
+        if access_token = find_access_token
+          case validate_access_token(access_token, scopes)
+          when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+            raise InsufficientScopeError.new(scopes)
 
-        when Oauth2::AccessTokenValidationService::EXPIRED
-          raise ExpiredError
+          when Oauth2::AccessTokenValidationService::EXPIRED
+            raise ExpiredError
 
-        when Oauth2::AccessTokenValidationService::REVOKED
-          raise RevokedError
+          when Oauth2::AccessTokenValidationService::REVOKED
+            raise RevokedError
 
-        when Oauth2::AccessTokenValidationService::VALID
-          @current_user = User.find(access_token.resource_owner_id)
+          when Oauth2::AccessTokenValidationService::VALID
+            @current_user = User.find(access_token.resource_owner_id)
+          end
         end
       end
-    end
 
-    def current_user
-      @current_user
-    end
+      def current_user
+        @current_user
+      end
 
-    private
-    def find_access_token
-      @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
-    end
+      private
 
-    def doorkeeper_request
-      @doorkeeper_request ||= ActionDispatch::Request.new(env)
-    end
+      def find_access_token
+        @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
+      end
 
-    def validate_access_token(access_token, scopes)
-      Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
-    end
-  end
+      def doorkeeper_request
+        @doorkeeper_request ||= ActionDispatch::Request.new(env)
+      end
 
-  module ClassMethods
-    # Installs the doorkeeper guard on the whole Grape API endpoint.
-    #
-    # Arguments:
-    #
-    #   scopes: (optional) scopes required for this guard.
-    #           Defaults to empty array.
-    #
-    def guard_all!(scopes: [])
-      before do
-        guard! scopes: scopes
+      def validate_access_token(access_token, scopes)
+        Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
       end
     end
 
-    private
-    def install_error_responders(base)
-      error_classes = [ MissingTokenError, TokenNotFoundError,
-                        ExpiredError, RevokedError, InsufficientScopeError]
+    module ClassMethods
+      # Installs the doorkeeper guard on the whole Grape API endpoint.
+      #
+      # Arguments:
+      #
+      #   scopes: (optional) scopes required for this guard.
+      #           Defaults to empty array.
+      #
+      def guard_all!(scopes: [])
+        before do
+          guard! scopes: scopes
+        end
+      end
 
-      base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
-    end
+      private
 
-    def oauth2_bearer_token_error_handler
-      Proc.new do |e|
-        response =
-          case e
-          when MissingTokenError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
-
-          when TokenNotFoundError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Bad Access Token.")
-
-          when ExpiredError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Token is expired. You can either do re-authorization or token refresh.")
-
-          when RevokedError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Token was revoked. You have to re-authorize from the user.")
-
-          when InsufficientScopeError
-            # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
-            # does not include WWW-Authenticate header, which breaks the standard.
-            Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
-              :insufficient_scope,
-              Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
-              { scope: e.scopes })
-          end
+      def install_error_responders(base)
+        error_classes = [ MissingTokenError, TokenNotFoundError,
+                          ExpiredError, RevokedError, InsufficientScopeError]
 
-        response.finish
+        base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
+      end
+
+      def oauth2_bearer_token_error_handler
+        Proc.new do |e|
+          response =
+            case e
+            when MissingTokenError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
+
+            when TokenNotFoundError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Bad Access Token.")
+
+            when ExpiredError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Token is expired. You can either do re-authorization or token refresh.")
+
+            when RevokedError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Token was revoked. You have to re-authorize from the user.")
+
+            when InsufficientScopeError
+              # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
+              # does not include WWW-Authenticate header, which breaks the standard.
+              Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
+                :insufficient_scope,
+                Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
+                { scope: e.scopes })
+            end
+
+          response.finish
+        end
       end
     end
-  end
 
-  #
-  # Exceptions
-  #
+    #
+    # Exceptions
+    #
 
-  class MissingTokenError < StandardError; end
+    class MissingTokenError < StandardError; end
 
-  class TokenNotFoundError < StandardError; end
+    class TokenNotFoundError < StandardError; end
 
-  class ExpiredError < StandardError; end
+    class ExpiredError < StandardError; end
 
-  class RevokedError < StandardError; end
+    class RevokedError < StandardError; end
 
-  class InsufficientScopeError < StandardError
-    attr_reader :scopes
-    def initialize(scopes)
-      @scopes = scopes
+    class InsufficientScopeError < StandardError
+      attr_reader :scopes
+      def initialize(scopes)
+        @scopes = scopes
+      end
     end
   end
 end
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 7388ed2f4ea..9bcd33ff19e 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -2,7 +2,7 @@ require 'mime/types'
 
 module API
   # Project commit statuses API
-  class CommitStatus < Grape::API
+  class CommitStatuses < Grape::API
     resource :projects do
       before { authenticate! }
 
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 353c4ddebf8..17bb99a2ae5 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -1,9 +1,7 @@
-Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
-
 module Ci
   module API
     class API < Grape::API
-      include APIGuard
+      include ::API::APIGuard
       version 'v1', using: :path
 
       rescue_from ActiveRecord::RecordNotFound do
@@ -31,9 +29,9 @@ module Ci
       helpers ::API::Helpers
       helpers Gitlab::CurrentSettings
 
-      mount Builds
-      mount Runners
-      mount Triggers
+      mount ::Ci::API::Builds
+      mount ::Ci::API::Runners
+      mount ::Ci::API::Triggers
     end
   end
 end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 7479e729db1..37f4c34054f 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,4 +1,4 @@
-require 'gitlab/git'
+require_dependency 'gitlab/git'
 
 module Gitlab
   def self.com?