From fd9377fa1a4e1eaef280200430c47036d0e58d71 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Sat, 4 Aug 2018 01:02:38 +0200 Subject: Added method to re-populating SiteStatitiscs counter --- app/models/site_statistic.rb | 16 +++++++++++++++- spec/models/site_statistic_spec.rb | 12 ++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/app/models/site_statistic.rb b/app/models/site_statistic.rb index daac1c57db9..2530a9d8b8e 100644 --- a/app/models/site_statistic.rb +++ b/app/models/site_statistic.rb @@ -49,7 +49,7 @@ class SiteStatistic < ActiveRecord::Base # # @return [SiteStatistic] record with tracked information def self.fetch - SiteStatistic.transaction(requires_new: true) do + transaction(requires_new: true) do SiteStatistic.first_or_create! end rescue ActiveRecord::RecordNotUnique @@ -73,4 +73,18 @@ class SiteStatistic < ActiveRecord::Base super end + + def self.recalculate_counters! + transaction do + # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? + self.update_all('repositories_count = (SELECT COUNT(*) FROM projects)') + end + + transaction do + # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? + self.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)') + end + end end diff --git a/spec/models/site_statistic_spec.rb b/spec/models/site_statistic_spec.rb index 9b056fbf332..49b54fb6994 100644 --- a/spec/models/site_statistic_spec.rb +++ b/spec/models/site_statistic_spec.rb @@ -80,4 +80,16 @@ describe SiteStatistic do end end end + + describe '.recalculate_counters!' do + it 'recalculates existing counters' do + create(:project) + described_class.fetch.update(repositories_count: 0, wikis_count: 0) + + described_class.recalculate_counters! + + expect(described_class.fetch.repositories_count).to eq(1) + expect(described_class.fetch.wikis_count).to eq(1) + end + end end -- cgit v1.2.1 From 85e83813379bb7be86e266c67dedcfee2cd6d738 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 9 Aug 2018 20:41:39 +0200 Subject: Add WikiAccessLevel migration from `NULL` to `20` --- ...180809195358_migrate_null_wiki_access_levels.rb | 23 +++++++++++++++++ db/schema.rb | 2 +- .../migrate_null_wiki_access_levels_spec.rb | 29 ++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb create mode 100644 spec/migrations/migrate_null_wiki_access_levels_spec.rb diff --git a/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb new file mode 100644 index 00000000000..d3557efecbd --- /dev/null +++ b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class MigrateNullWikiAccessLevels < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + class ProjectFeature < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_features' + end + + def up + ProjectFeature.where(wiki_access_level: nil).each_batch do |relation| + relation.update_all(wiki_access_level: 20) + end + end + + def down + # there is no way to rollback this change, there are no downsides in keeping migrated data. + end +end diff --git a/db/schema.rb b/db/schema.rb index 1288a98745c..9dc122b54b3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180808162000) do +ActiveRecord::Schema.define(version: 20180809195358) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/spec/migrations/migrate_null_wiki_access_levels_spec.rb b/spec/migrations/migrate_null_wiki_access_levels_spec.rb new file mode 100644 index 00000000000..f99273072a2 --- /dev/null +++ b/spec/migrations/migrate_null_wiki_access_levels_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20180809195358_migrate_null_wiki_access_levels.rb') + +describe MigrateNullWikiAccessLevels, :migration do + let(:namespaces) { table('namespaces') } + let(:projects) { table(:projects) } + let(:project_features) { table(:project_features) } + let(:migration) { described_class.new } + + before do + namespace = namespaces.create(name: 'foo', path: 'foo') + + projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: namespace.id) + projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2', namespace_id: namespace.id) + projects.create!(id: 3, name: 'gitlab3', path: 'gitlab3', namespace_id: namespace.id) + + project_features.create!(id: 1, project_id: 1, wiki_access_level: nil) + project_features.create!(id: 2, project_id: 2, wiki_access_level: 10) + project_features.create!(id: 3, project_id: 3, wiki_access_level: 20) + end + + describe '#up' do + it 'migrates existing project_features with wiki_access_level NULL to 20' do + expect { migration.up }.to change { project_features.where(wiki_access_level: 20).count }.by(1) + end + end +end -- cgit v1.2.1 From 010fbedb61334a303d26164de905d9d421871ff2 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Thu, 9 Aug 2018 21:13:14 +0200 Subject: Changelog --- changelogs/unreleased/repopulate_site_statistics.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/repopulate_site_statistics.yml diff --git a/changelogs/unreleased/repopulate_site_statistics.yml b/changelogs/unreleased/repopulate_site_statistics.yml new file mode 100644 index 00000000000..1961088061d --- /dev/null +++ b/changelogs/unreleased/repopulate_site_statistics.yml @@ -0,0 +1,5 @@ +--- +title: Migrate NULL wiki_access_level to correct number so we count active wikis correctly +merge_request: 21030 +author: +type: changed -- cgit v1.2.1 From 696a5fce687364ec718eeef44d923a686c9e1624 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 15 Aug 2018 01:39:59 +0200 Subject: Refactor Rainbow usage in specs We don't test any specific string generated by it yet, so there is no point in keeping it enabled when we are disabling it everywhere we test its output. --- spec/lib/gitlab/bare_repository_import/importer_spec.rb | 4 ---- spec/lib/system_check/simple_executor_spec.rb | 9 --------- spec/spec_helper.rb | 1 + 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb index 6e21c846c0a..3c63e601abc 100644 --- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -10,9 +10,6 @@ describe Gitlab::BareRepositoryImport::Importer, :seed_helper do subject(:importer) { described_class.new(admin, bare_repository) } before do - @rainbow = Rainbow.enabled - Rainbow.enabled = false - allow(described_class).to receive(:log) end @@ -20,7 +17,6 @@ describe Gitlab::BareRepositoryImport::Importer, :seed_helper do FileUtils.rm_rf(base_dir) TestEnv.clean_test_path ensure_seeds - Rainbow.enabled = @rainbow end shared_examples 'importing a repository' do diff --git a/spec/lib/system_check/simple_executor_spec.rb b/spec/lib/system_check/simple_executor_spec.rb index 9da3648400e..e71e9da369d 100644 --- a/spec/lib/system_check/simple_executor_spec.rb +++ b/spec/lib/system_check/simple_executor_spec.rb @@ -98,15 +98,6 @@ describe SystemCheck::SimpleExecutor do end end - before do - @rainbow = Rainbow.enabled - Rainbow.enabled = false - end - - after do - Rainbow.enabled = @rainbow - end - describe '#component' do it 'returns stored component name' do expect(subject.component).to eq('Test') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f4441a6b700..a15a46a9534 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -29,6 +29,7 @@ end # require rainbow gem String monkeypatch, so we can test SystemChecks require 'rainbow/ext/string' +Rainbow.enabled = false # Requires supporting ruby files with custom matchers and macros, etc, # in spec/support/ and its subdirectories. -- cgit v1.2.1 From 1a54986c166fb13a6a27afcafaa055e1a1749e38 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 15 Aug 2018 01:40:29 +0200 Subject: Refactor SiteStatistics to extract refresh logic into a rake task --- app/models/site_statistic.rb | 14 -------------- lib/tasks/gitlab/site_statistics.rake | 23 +++++++++++++++++++++++ spec/models/site_statistic_spec.rb | 12 ------------ spec/tasks/gitlab/site_statistics_rake_spec.rb | 24 ++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 lib/tasks/gitlab/site_statistics.rake create mode 100644 spec/tasks/gitlab/site_statistics_rake_spec.rb diff --git a/app/models/site_statistic.rb b/app/models/site_statistic.rb index 2530a9d8b8e..48324570f0b 100644 --- a/app/models/site_statistic.rb +++ b/app/models/site_statistic.rb @@ -73,18 +73,4 @@ class SiteStatistic < ActiveRecord::Base super end - - def self.recalculate_counters! - transaction do - # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 - ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? - self.update_all('repositories_count = (SELECT COUNT(*) FROM projects)') - end - - transaction do - # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 - ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? - self.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)') - end - end end diff --git a/lib/tasks/gitlab/site_statistics.rake b/lib/tasks/gitlab/site_statistics.rake new file mode 100644 index 00000000000..7d24ec72a9d --- /dev/null +++ b/lib/tasks/gitlab/site_statistics.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + desc "GitLab | Refresh Site Statistics counters" + task refresh_site_statistics: :environment do + puts 'Updating Site Statistics counters: ' + + print '* Repositories... ' + SiteStatistic.transaction do + # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? + SiteStatistic.update_all('repositories_count = (SELECT COUNT(*) FROM projects)') + end + puts 'OK!'.color(:green) + + print '* Wikis... ' + SiteStatistic.transaction do + # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? + SiteStatistic.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)') + end + puts 'OK!'.color(:green) + puts + end +end diff --git a/spec/models/site_statistic_spec.rb b/spec/models/site_statistic_spec.rb index 49b54fb6994..9b056fbf332 100644 --- a/spec/models/site_statistic_spec.rb +++ b/spec/models/site_statistic_spec.rb @@ -80,16 +80,4 @@ describe SiteStatistic do end end end - - describe '.recalculate_counters!' do - it 'recalculates existing counters' do - create(:project) - described_class.fetch.update(repositories_count: 0, wikis_count: 0) - - described_class.recalculate_counters! - - expect(described_class.fetch.repositories_count).to eq(1) - expect(described_class.fetch.wikis_count).to eq(1) - end - end end diff --git a/spec/tasks/gitlab/site_statistics_rake_spec.rb b/spec/tasks/gitlab/site_statistics_rake_spec.rb new file mode 100644 index 00000000000..20f0df65e63 --- /dev/null +++ b/spec/tasks/gitlab/site_statistics_rake_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +require 'rake_helper' + +describe 'rake gitlab:refresh_site_statistics' do + before do + Rake.application.rake_require 'tasks/gitlab/site_statistics' + + create(:project) + SiteStatistic.fetch.update(repositories_count: 0, wikis_count: 0) + end + + let(:task) { 'gitlab:refresh_site_statistics' } + + it 'recalculates existing counters' do + run_rake_task(task) + + expect(SiteStatistic.fetch.repositories_count).to eq(1) + expect(SiteStatistic.fetch.wikis_count).to eq(1) + end + + it 'displays message listing counters' do + expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!.* Wikis\.\.\. OK!/m).to_stdout + end +end -- cgit v1.2.1 From 7336bd4843521956997c80a10705a83083eb122b Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 15 Aug 2018 04:38:39 +0200 Subject: Add site statistics recount to the post migiration --- .../20180809195358_migrate_null_wiki_access_levels.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb index d3557efecbd..0a0a33299e4 100644 --- a/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb +++ b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb @@ -5,6 +5,8 @@ class MigrateNullWikiAccessLevels < ActiveRecord::Migration DOWNTIME = false + disable_ddl_transaction! + class ProjectFeature < ActiveRecord::Base include EachBatch @@ -15,6 +17,13 @@ class MigrateNullWikiAccessLevels < ActiveRecord::Migration ProjectFeature.where(wiki_access_level: nil).each_batch do |relation| relation.update_all(wiki_access_level: 20) end + + # We need to re-count wikis as previous attempt was not considering the NULLs. + transaction do + execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + + execute("UPDATE site_statistics SET wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)") + end end def down -- cgit v1.2.1