elastic: almost all integrated exept view logicelasticsearch

13 files changed, 58 insertions, 85 deletions

diff --git a/Gemfile b/Gemfile
index 654cff7072a..b5738e3d78a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -94,6 +94,10 @@ gem "six", '~> 0.2.0'
 # Seed data
 gem "seed-fu", '~> 2.3.5'
 
+# Search
+gem 'elasticsearch-model',  github: 'elasticsearch/elasticsearch-rails',  ref: '88b6597e47c9f45024b603faeddb0a85b47e1fce'
+gem 'elasticsearch-rails',  github: 'elasticsearch/elasticsearch-rails'
+
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'
 gem 'task_list',     '~> 1.0.2', require: 'task_list/railtie'
diff --git a/Gemfile.lock b/Gemfile.lock
index 9e920414046..475b7b176fd 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,19 @@
+GIT
+  remote: git://github.com/elasticsearch/elasticsearch-rails.git
+  revision: 88b6597e47c9f45024b603faeddb0a85b47e1fce
+  ref: 88b6597e47c9f45024b603faeddb0a85b47e1fce
+  specs:
+    elasticsearch-model (0.1.0.rc1)
+      activesupport (> 3)
+      elasticsearch (> 0.4)
+      hashie
+
+GIT
+  remote: git://github.com/elasticsearch/elasticsearch-rails.git
+  revision: 5f32e484a6d0458f26dc9acfa9dd5fed4e5d6453
+  specs:
+    elasticsearch-rails (0.1.0.rc1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -177,6 +193,14 @@ GEM
       railties (>= 3.2)
     dropzonejs-rails (0.7.2)
       rails (> 3.1)
+    elasticsearch (1.0.15)
+      elasticsearch-api (= 1.0.15)
+      elasticsearch-transport (= 1.0.15)
+    elasticsearch-api (1.0.15)
+      multi_json
+    elasticsearch-transport (1.0.15)
+      faraday
+      multi_json
     email_reply_parser (0.5.8)
     email_spec (1.6.0)
       launchy (~> 2.1)
@@ -858,6 +882,8 @@ DEPENDENCIES
   diffy (~> 3.0.3)
   doorkeeper (~> 2.2.0)
   dropzonejs-rails (~> 0.7.1)
+  elasticsearch-model!
+  elasticsearch-rails!
   email_reply_parser (~> 0.5.8)
   email_spec (~> 1.6.0)
   enumerize (~> 0.7.0)
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
index 68dfc4d571c..542e2397504 100644
--- a/app/controllers/search_controller.rb
+++ b/app/controllers/search_controller.rb
@@ -48,9 +48,9 @@ class SearchController < ApplicationController
 
     if project
       return access_denied! unless can?(current_user, :download_code, project)
-      @search_results = SearchService.new(current_user, params).project_search(project)
+      @search_results = SearchService.new(nil, current_user, params).project_search(project)
     else
-      @search_results = SearchService.new(current_user, params).global_search
+      @search_results = SearchService.new(nil, current_user, params).global_search
     end
 
     @search_results = SearchDecorator.new(@search_results, params[:type])
diff --git a/app/elastic/application_search.rb b/app/elastic/application_search.rb
index 32a96f11a60..44577e25963 100644
--- a/app/elastic/application_search.rb
+++ b/app/elastic/application_search.rb
@@ -46,10 +46,10 @@ module ApplicationSearch
       }
     }
 
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index,  self.class.to_s, self.id) }, on: :create
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
-    after_touch  lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index,  self.class.to_s, self.id) }, on: :create
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
+    # after_touch  lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
   end
 
    module ClassMethods
diff --git a/app/elastic/groups_search.rb b/app/elastic/groups_search.rb
index 35fbc4a617a..2252dd254c4 100644
--- a/app/elastic/groups_search.rb
+++ b/app/elastic/groups_search.rb
@@ -24,7 +24,7 @@ module GroupsSearch
       })
     end
 
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
 
       page ||= 1
 
diff --git a/app/elastic/issues_search.rb b/app/elastic/issues_search.rb
index 946a0eae892..1b7318b711d 100644
--- a/app/elastic/issues_search.rb
+++ b/app/elastic/issues_search.rb
@@ -41,7 +41,7 @@ module IssuesSearch
       })
     end
 
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
 
       page ||= 1
 
diff --git a/app/elastic/merge_requests_search.rb b/app/elastic/merge_requests_search.rb
index 175c0c50142..eb0970b06d7 100644
--- a/app/elastic/merge_requests_search.rb
+++ b/app/elastic/merge_requests_search.rb
@@ -47,7 +47,7 @@ module MergeRequestsSearch
       })
     end
 
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
 
       page ||= 1
 
diff --git a/app/elastic/projects_search.rb b/app/elastic/projects_search.rb
index 8236698590b..b9620cb6616 100644
--- a/app/elastic/projects_search.rb
+++ b/app/elastic/projects_search.rb
@@ -54,7 +54,7 @@ module ProjectsSearch
       })
     end
 
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
 
       page ||= 1
 
diff --git a/app/elastic/repositories_search.rb b/app/elastic/repositories_search.rb
deleted file mode 100644
index d97a73241be..00000000000
--- a/app/elastic/repositories_search.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-module RepositoriesSearch
-  extend ActiveSupport::Concern
-
-  included do
-    include Elasticsearch::Git::Repository
-
-    self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
-
-    def repository_id
-      project.id
-    end
-
-    def self.repositories_count
-      Project.count
-    end
-
-    def client_for_indexing
-      self.__elasticsearch__.client
-    end
-
-    def self.import
-      Repository.__elasticsearch__.create_index! force: true
-
-      Project.find_each do |project|
-        if project.repository.exists? && !project.repository.empty?
-          begin
-            project.repository.index_commits
-          rescue
-          end
-          begin
-            project.repository.index_blobs
-          rescue
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/app/elastic/users_search.rb b/app/elastic/users_search.rb
index 9e86f77a7ad..60b5ddaa4b2 100644
--- a/app/elastic/users_search.rb
+++ b/app/elastic/users_search.rb
@@ -31,7 +31,7 @@ module UsersSearch
       })
     end
 
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
 
       page ||= 1
       per ||= 20
diff --git a/app/models/repository.rb b/app/models/repository.rb
index a2d4fcb93de..b8f4a26bb19 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -1,8 +1,6 @@
 require 'securerandom'
 
 class Repository
-  include RepositoriesSearch
-
   class CommitError < StandardError; end
 
   MIRROR_REMOTE = "upstream"
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 22e15fdc78a..f31794a96e7 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -7,20 +7,19 @@ class SearchService < BaseService
         users: search_in_users(query),
         projects: search_in_projects(query),
         merge_requests: search_in_merge_requests(query),
-        issues: search_in_issues(query),
-        repositories: search_in_repository(query),
+        issues: search_in_issues(query)
     }
   end
 
   def project_search(project)
     query = params[:search]
+
     {
         groups: {},
         users: {},
         projects: {},
         merge_requests: search_in_merge_requests(query, project),
-        issues: search_in_issues(query, project),
-        repositories: search_in_repository(query, project),
+        issues: search_in_issues(query, project)
     }
   end
 
@@ -36,12 +35,13 @@ class SearchService < BaseService
     }
 
     group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
+
     opt[:namespace_id] = group.id if group
 
     opt[:category] = params[:category] if params[:category].present?
 
     begin
-      response = Project.search(query, options: opt, page: page)
+      response = Project.elastic_search(query, options: opt, page: page)
 
       categories_list = if query.blank?
                           Project.category_counts.map do |category|
@@ -75,7 +75,7 @@ class SearchService < BaseService
     }
 
     begin
-      response = Group.search(query, options: opt, page: page)
+      response = Group.elastic_search(query, options: opt, page: page)
 
       {
         records: response.records,
@@ -96,7 +96,7 @@ class SearchService < BaseService
     }
 
     begin
-      response = User.search(query, options: opt, page: page)
+      response = User.elastic_search(query, options: opt, page: page)
 
       {
         records: response.records,
@@ -117,7 +117,7 @@ class SearchService < BaseService
     }
 
     begin
-      response = MergeRequest.search(query, options: opt, page: page)
+      response = MergeRequest.elastic_search(query, options: opt, page: page)
 
       {
         records: response.records,
@@ -137,7 +137,7 @@ class SearchService < BaseService
     }
 
     begin
-      response = Issue.search(query, options: opt, page: page)
+      response = Issue.elastic_search(query, options: opt, page: page)
 
       {
         records: response.records,
@@ -150,33 +150,11 @@ class SearchService < BaseService
     end
   end
 
-  def search_in_repository(query, project = nil)
-    opt = {
-      repository_id: project ? [project.id] : projects_ids,
-      highlight: true,
-      order: params[:order]
-    }
-
-    if params[:language].present? && params[:language] != 'All'
-      opt.merge!({ language: params[:language] })
-    end
-
-    begin
-      res = Repository.search(query, options: opt, page: page)
-
-      res[:blobs][:projects] = project_filter(res[:blobs][:repositories]) || []
-      res[:commits][:projects]  = project_filter(res[:commits][:repositories]) || []
-
-      res
-    rescue Exception => e
-      {}
-    end
-  end
-
   def projects_ids
     @allowed_projects_ids ||= begin
       if params[:namespace].present?
         namespace = Namespace.find_by(path: params[:namespace])
+
         if namespace
           return namespace.projects.where(id: known_projects_ids).pluck(:id)
         end
@@ -196,8 +174,8 @@ class SearchService < BaseService
 
   def known_projects_ids
     known_projects_ids = []
-    known_projects_ids += current_user.known_projects.pluck(:id) if current_user
-    known_projects_ids + Project.public_or_internal_only(current_user).pluck(:id)
+    known_projects_ids += current_user.authorized_projects.pluck(:id) if current_user
+    known_projects_ids + Project.public_and_internal_only.pluck(:id)
   end
 
   def project_filter(es_results)
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 16e92ac7080..96a77d7250b 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -136,6 +136,11 @@ production: &base
     # The location where LFS objects are stored (default: shared/lfs-objects).
     # storage_path: shared/lfs-objects
 
+  ## Elasticsearch (EE only)
+  elasticsearch:
+    host: localhost
+    port: 9200
+
   ## GitLab Pages
   pages:
     enabled: false