diff options
author | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2015-07-17 15:04:49 +0000 |
---|---|---|
committer | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2015-07-17 17:22:13 +0200 |
commit | 2fa3acaa4376ddb86144cb5e76fdd0bfa32af1fb (patch) | |
tree | 7c6d8281d1260189640e7b600b74452877022233 | |
parent | ae44e5ec064aaa9f5063732dc7bb6ee751a80ce2 (diff) | |
download | gitlab-ce-2fa3acaa4376ddb86144cb5e76fdd0bfa32af1fb.tar.gz |
Merge branch 'advanced-cache' into 'master'
Advanced cache
Fixes #1993
* Build missing cache values in background job after each push
* Store commit_count in project table. Refresh in background job
* moved repository size calculation in background job
## Advantages
Every time push is triggered - we build cache for project even without user visiting project page.
That means first visit is as fast as others now. This is especially good for active projects where people have some requests fast because of cache and some slow - because cache was build in their request.
Between cache expired and cache built we we had gap when Linux repo can give 502 error because calculation commits count takes 30 seconds or even more.
Using value from database fix this problem. Before cache is updated you see old value from database. After - you see new one.
Basically this merge request is super win to GitLab. We don't do some heavy operations in user request but instead in background job.
## Temporary problem
After this migration all projects will have `0` commits in database. It fill be replaced with real value on next push. I did not add recalculation to migration because it will last forever on big instances. Can be fixed by adding rake task which will go in background on live instance without downtime
See merge request !986
-rw-r--r-- | app/helpers/projects_helper.rb | 14 | ||||
-rw-r--r-- | app/models/project.rb | 4 | ||||
-rw-r--r-- | app/models/repository.rb | 35 | ||||
-rw-r--r-- | app/services/git_push_service.rb | 2 | ||||
-rw-r--r-- | app/services/git_tag_push_service.rb | 6 | ||||
-rw-r--r-- | app/views/explore/projects/_project.html.haml | 2 | ||||
-rw-r--r-- | app/views/projects/show.html.haml | 2 | ||||
-rw-r--r-- | app/workers/project_cache_worker.rb | 15 | ||||
-rw-r--r-- | app/workers/repository_import_worker.rb | 2 | ||||
-rw-r--r-- | db/fixtures/development/04_project.rb | 35 | ||||
-rw-r--r-- | db/migrate/20150717130904_add_commits_count_to_project.rb | 5 | ||||
-rw-r--r-- | db/schema.rb | 3 | ||||
-rw-r--r-- | lib/repository_cache.rb | 8 |
13 files changed, 110 insertions, 23 deletions
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 78f24dbd7ef..f61baf00525 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -276,4 +276,18 @@ module ProjectsHelper def readme_cache_key [@project.id, @project.commit.sha, "readme"].join('-') end + + def round_commit_count(project) + count = project.commit_count + + if count > 10000 + '10000+' + elsif count > 5000 + '5000+' + elsif count > 1000 + '1000+' + else + count + end + end end diff --git a/app/models/project.rb b/app/models/project.rb index b161cbe86b9..ff372ea9aa5 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -683,6 +683,10 @@ class Project < ActiveRecord::Base update_attribute(:repository_size, repository.size) end + def update_commit_count + update_attribute(:commit_count, repository.commit_count) + end + def forks_count ForkedProjectLink.where(forked_from_project_id: self.id).count end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6262b5c4c92..2985619fd2e 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -94,18 +94,6 @@ class Repository gitlab_shell.rm_tag(path_with_namespace, tag_name) end - def round_commit_count - if commit_count > 10000 - '10000+' - elsif commit_count > 5000 - '5000+' - elsif commit_count > 1000 - '1000+' - else - commit_count - end - end - def branch_names cache.fetch(:branch_names) { raw_repository.branch_names } end @@ -130,10 +118,29 @@ class Repository cache.fetch(:size) { raw_repository.size } end - def expire_cache + def cache_keys %i(size branch_names tag_names commit_count graph_log - readme version contribution_guide changelog license).each do |key| + readme version contribution_guide changelog license) + end + + def build_cache + cache_keys.each do |key| + unless cache.exist?(key) + send(key) + end + end + end + + def expire_cache + cache_keys.each do |key| + cache.expire(key) + end + end + + def rebuild_cache + cache_keys.each do |key| cache.expire(key) + send(key) end end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index 6135ae65007..3511392d1d8 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -21,7 +21,6 @@ class GitPushService project.ensure_satellite_exists project.repository.expire_cache - project.update_repository_size if push_remove_branch?(ref, newrev) @push_commits = [] @@ -61,6 +60,7 @@ class GitPushService EventCreateService.new.push(project, user, @push_data) project.execute_hooks(@push_data.dup, :push_hooks) project.execute_services(@push_data.dup, :push_hooks) + ProjectCacheWorker.perform_async(project.id) end protected diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb index 075a6118da2..1cc42b0b0ad 100644 --- a/app/services/git_tag_push_service.rb +++ b/app/services/git_tag_push_service.rb @@ -2,15 +2,15 @@ class GitTagPushService attr_accessor :project, :user, :push_data def execute(project, user, oldrev, newrev, ref) - @project, @user = project, user + project.repository.expire_cache + @project, @user = project, user @push_data = build_push_data(oldrev, newrev, ref) EventCreateService.new.push(project, user, @push_data) project.execute_hooks(@push_data.dup, :tag_push_hooks) project.execute_services(@push_data.dup, :tag_push_hooks) - - project.repository.expire_cache + ProjectCacheWorker.perform_async(project.id) true end diff --git a/app/views/explore/projects/_project.html.haml b/app/views/explore/projects/_project.html.haml index d65fb529373..d769c91545d 100644 --- a/app/views/explore/projects/_project.html.haml +++ b/app/views/explore/projects/_project.html.haml @@ -14,7 +14,7 @@ .repo-info - unless project.empty_repo? - = link_to pluralize(project.repository.round_commit_count, 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch) + = link_to pluralize(round_commit_count(project), 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch) · = link_to pluralize(project.repository.branch_names.count, 'branch'), namespace_project_branches_path(project.namespace, project) · diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml index 98d9053eb1d..b90cadfb1e2 100644 --- a/app/views/projects/show.html.haml +++ b/app/views/projects/show.html.haml @@ -13,7 +13,7 @@ %ul.nav.nav-pills %li = link_to namespace_project_commits_path(@project.namespace, @project, @ref || @repository.root_ref) do - = pluralize(number_with_delimiter(@repository.commit_count), 'commit') + = pluralize(number_with_delimiter(@project.commit_count), 'commit') %li = link_to namespace_project_branches_path(@project.namespace, @project) do = pluralize(number_with_delimiter(@repository.branch_names.count), 'branch') diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb new file mode 100644 index 00000000000..55cb6af232e --- /dev/null +++ b/app/workers/project_cache_worker.rb @@ -0,0 +1,15 @@ +class ProjectCacheWorker + include Sidekiq::Worker + + sidekiq_options queue: :default + + def perform(project_id) + project = Project.find(project_id) + project.update_repository_size + project.update_commit_count + + if project.repository.root_ref + project.repository.build_cache + end + end +end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index e6a50afedb1..94832872d13 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -28,7 +28,7 @@ class RepositoryImportWorker project.import_finish project.save project.satellite.create unless project.satellite.exists? - project.update_repository_size + ProjectCacheWorker.perform_async(project.id) Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket' end end diff --git a/db/fixtures/development/04_project.rb b/db/fixtures/development/04_project.rb index 87839770924..8f71198e47f 100644 --- a/db/fixtures/development/04_project.rb +++ b/db/fixtures/development/04_project.rb @@ -11,9 +11,42 @@ Sidekiq::Testing.inline! do 'https://github.com/twitter/flight.git', 'https://github.com/twitter/typeahead.js.git', 'https://github.com/h5bp/html5-boilerplate.git', + 'https://github.com/google/material-design-lite.git', + 'https://github.com/jlevy/the-art-of-command-line.git', + 'https://github.com/FreeCodeCamp/freecodecamp.git', + 'https://github.com/google/deepdream.git', + 'https://github.com/jtleek/datasharing.git', + 'https://github.com/WebAssembly/design.git', + 'https://github.com/airbnb/javascript.git', + 'https://github.com/tessalt/echo-chamber-js.git', + 'https://github.com/atom/atom.git', + 'https://github.com/ipselon/react-ui-builder.git', + 'https://github.com/mattermost/platform.git', + 'https://github.com/purifycss/purifycss.git', + 'https://github.com/facebook/nuclide.git', + 'https://github.com/wbkd/awesome-d3.git', + 'https://github.com/kilimchoi/engineering-blogs.git', + 'https://github.com/gilbarbara/logos.git', + 'https://github.com/gaearon/redux.git', + 'https://github.com/awslabs/s2n.git', + 'https://github.com/arkency/reactjs_koans.git', + 'https://github.com/twbs/bootstrap.git', + 'https://github.com/chjj/ttystudio.git', + 'https://github.com/DrBoolean/mostly-adequate-guide.git', + 'https://github.com/octocat/Spoon-Knife.git', + 'https://github.com/opencontainers/runc.git', + 'https://github.com/googlesamples/android-topeka.git' ] - project_urls.each_with_index do |url, i| + # You can specify how many projects you need during seed execution + size = if ENV['SIZE'].present? + ENV['SIZE'].to_i + else + 8 + end + + + project_urls.first(size).each_with_index do |url, i| group_path, project_path = url.split('/')[-2..-1] group = Group.find_by(path: group_path) diff --git a/db/migrate/20150717130904_add_commits_count_to_project.rb b/db/migrate/20150717130904_add_commits_count_to_project.rb new file mode 100644 index 00000000000..9b46daa5933 --- /dev/null +++ b/db/migrate/20150717130904_add_commits_count_to_project.rb @@ -0,0 +1,5 @@ +class AddCommitsCountToProject < ActiveRecord::Migration + def change + add_column :projects, :commit_count, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index d6c34a77ee6..a63c2d05821 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150713160110) do +ActiveRecord::Schema.define(version: 20150717130904) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -374,6 +374,7 @@ ActiveRecord::Schema.define(version: 20150713160110) do t.integer "star_count", default: 0, null: false t.string "import_type" t.string "import_source" + t.integer "commit_count", default: 0 end add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree diff --git a/lib/repository_cache.rb b/lib/repository_cache.rb index fa016a170cd..8ddc3511293 100644 --- a/lib/repository_cache.rb +++ b/lib/repository_cache.rb @@ -18,4 +18,12 @@ class RepositoryCache def fetch(key, &block) backend.fetch(cache_key(key), &block) end + + def exist?(key) + backend.exist?(cache_key(key)) + end + + def read(key) + backend.read(cache_key(key)) + end end |