From a7702271391524262788accfc78e6ef58b63f88e Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Mon, 6 Feb 2017 12:22:55 +0800 Subject: Remove inactive default email services Note that we no longer generate this by default. This is for clearing legacy default data. --- .../remove-inactive-default-email-services.yml | 4 ++++ ...40400_remove_inactive_default_email_services.rb | 27 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 changelogs/unreleased/remove-inactive-default-email-services.yml create mode 100644 db/post_migrate/20170206040400_remove_inactive_default_email_services.rb diff --git a/changelogs/unreleased/remove-inactive-default-email-services.yml b/changelogs/unreleased/remove-inactive-default-email-services.yml new file mode 100644 index 00000000000..c32c1390e4e --- /dev/null +++ b/changelogs/unreleased/remove-inactive-default-email-services.yml @@ -0,0 +1,4 @@ +--- +title: Remove inactive default email services +merge_request: 8987 +author: diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb new file mode 100644 index 00000000000..18affebde73 --- /dev/null +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -0,0 +1,27 @@ +class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + execute <<-SQL.strip_heredoc + DELETE FROM services + WHERE type = 'BuildsEmailService' + AND active = #{false_value} + AND properties = '{"notify_only_broken_builds":true}'; + + DELETE FROM services + WHERE type = 'PipelinesEmailService' + AND active = #{false_value} + AND properties = '{"notify_only_broken_pipelines":true}'; + SQL + end + + def false_value + quote(false) + end + + def quote(value) + ActiveRecord::Base.connection.quote(value) + end +end -- cgit v1.2.1 From 25cd5aa228ebe10ce9eabb17c75eb86e9d8c152c Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Mon, 6 Feb 2017 12:39:36 +0800 Subject: Run two threads to improve migration running time --- ...6040400_remove_inactive_default_email_services.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index 18affebde73..dc7750f3244 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -4,24 +4,38 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration DOWNTIME = false def up - execute <<-SQL.strip_heredoc + builds_service = spawn <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'BuildsEmailService' AND active = #{false_value} AND properties = '{"notify_only_broken_builds":true}'; + SQL + pipelines_service = spawn <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'PipelinesEmailService' AND active = #{false_value} AND properties = '{"notify_only_broken_pipelines":true}'; SQL + + [builds_service, pipelines_service].each(&:join) end - def false_value - quote(false) + private + + def spawn(query) + Thread.new do + ActiveRecord::Base.connection_pool.with_connection do + ActiveRecord::Base.connection.execute(query) + end + end end def quote(value) ActiveRecord::Base.connection.quote(value) end + + def false_value + quote(false) + end end -- cgit v1.2.1 From 887aeefba63429b6603c75e385148a2c6be34be1 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Mon, 6 Feb 2017 21:51:19 +0800 Subject: Use IS FALSE for both pg and mysql; Handle connections by ourselves so that even if the setting has 1 connection we could still use more connections. --- ...6040400_remove_inactive_default_email_services.rb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index dc7750f3244..b107d9204d2 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -7,14 +7,14 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration builds_service = spawn <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'BuildsEmailService' - AND active = #{false_value} + AND active IS FALSE AND properties = '{"notify_only_broken_builds":true}'; SQL pipelines_service = spawn <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'PipelinesEmailService' - AND active = #{false_value} + AND active IS FALSE AND properties = '{"notify_only_broken_pipelines":true}'; SQL @@ -25,17 +25,19 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration def spawn(query) Thread.new do - ActiveRecord::Base.connection_pool.with_connection do - ActiveRecord::Base.connection.execute(query) + with_connection do |connection| + connection.execute(query) end end end - def quote(value) - ActiveRecord::Base.connection.quote(value) - end + def with_connection + pool = ActiveRecord::Base.establish_connection + connection = pool.connection + + yield(connection) - def false_value - quote(false) + ensure + connection.close end end -- cgit v1.2.1 From 521a7cafd4226a03d7f0b5d35a2370559fb345cb Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 7 Feb 2017 00:01:56 +0800 Subject: Try this way to provide database connection --- .../20170206040400_remove_inactive_default_email_services.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index b107d9204d2..d69f68a13d2 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -31,13 +31,10 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration end end - def with_connection + def with_connection(&block) pool = ActiveRecord::Base.establish_connection - connection = pool.connection - - yield(connection) - + pool.with_connection(&block) ensure - connection.close + pool.disconnect! end end -- cgit v1.2.1 From 0c2f4a3c422c522bd32bdcf36425e836ebee8ea6 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 7 Feb 2017 19:29:17 +0800 Subject: Bump schema to pass `rake db:setup` --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index d421d5c6774..57e3f2f6c54 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170210075922) do +ActiveRecord::Schema.define(version: 20170206040400) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 7ecee7a4d79718bc46086ee8dec23a00cea39b39 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 7 Feb 2017 22:09:13 +0800 Subject: Introduce ThreadedConnectionPool for customized pool This way we could reuse this pool for other migrations Feedback: * https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8987#note_22923350 * https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8987#note_22923365 --- ...40400_remove_inactive_default_email_services.rb | 45 ++++++++------------ lib/gitlab/database/threaded_connection_pool.rb | 48 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 lib/gitlab/database/threaded_connection_pool.rb diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index d69f68a13d2..52e7f91bb01 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -3,38 +3,27 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration DOWNTIME = false - def up - builds_service = spawn <<-SQL.strip_heredoc - DELETE FROM services - WHERE type = 'BuildsEmailService' - AND active IS FALSE - AND properties = '{"notify_only_broken_builds":true}'; - SQL - - pipelines_service = spawn <<-SQL.strip_heredoc - DELETE FROM services - WHERE type = 'PipelinesEmailService' - AND active IS FALSE - AND properties = '{"notify_only_broken_pipelines":true}'; - SQL + disable_ddl_transaction! - [builds_service, pipelines_service].each(&:join) - end - - private + def up + Gitlab::Database::ThreadedConnectionPool.with_pool(2) do |pool| + pool.execute_async <<-SQL.strip_heredoc + DELETE FROM services + WHERE type = 'BuildsEmailService' + AND active IS FALSE + AND properties = '{"notify_only_broken_builds":true}'; + SQL - def spawn(query) - Thread.new do - with_connection do |connection| - connection.execute(query) - end + pool.execute_async <<-SQL.strip_heredoc + DELETE FROM services + WHERE type = 'PipelinesEmailService' + AND active IS FALSE + AND properties = '{"notify_only_broken_pipelines":true}'; + SQL end end - def with_connection(&block) - pool = ActiveRecord::Base.establish_connection - pool.with_connection(&block) - ensure - pool.disconnect! + def down + # Nothing can be done to restore the records end end diff --git a/lib/gitlab/database/threaded_connection_pool.rb b/lib/gitlab/database/threaded_connection_pool.rb new file mode 100644 index 00000000000..1316b005741 --- /dev/null +++ b/lib/gitlab/database/threaded_connection_pool.rb @@ -0,0 +1,48 @@ +module Gitlab + module Database + class ThreadedConnectionPool + def self.with_pool(pool_size) + pool = new(pool_size) + + yield(pool) + + ensure + pool.join + pool.close + end + + def initialize(pool_size) + config = ActiveRecord::Base.configurations[Rails.env] + @ar_pool = ActiveRecord::Base.establish_connection( + config.merge(pool: pool_size)) + @workers = [] + @mutex = Mutex.new + end + + def execute_async(sql) + @mutex.synchronize do + @workers << Thread.new do + @ar_pool.with_connection do |connection| + connection.execute(sql) + end + end + end + end + + def join + threads = nil + + @mutex.synchronize do + threads = @workers.dup + @workers.clear + end + + threads.each(&:join) + end + + def close + @ar_pool.disconnect! + end + end + end +end -- cgit v1.2.1 From 1d8db6652f635d3f5326071b9dd00621b87e228b Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 9 Feb 2017 21:11:15 +0800 Subject: Revert "Bump schema to pass `rake db:setup`" This reverts commit f2ed7cbc9b36b6ad9bcc714b271e98ead756b816. --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 57e3f2f6c54..de07b3837ad 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170206040400) do +ActiveRecord::Schema.define(version: 20170204181513) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1 From 8aa1055fe3d24aa606f6a8d3c635f97b788e4e85 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Thu, 9 Feb 2017 21:11:31 +0800 Subject: Use threads directly, introduce pool later: Feedback: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/8987#note_22938402 --- ...40400_remove_inactive_default_email_services.rb | 38 ++++++++++++++--- lib/gitlab/database/threaded_connection_pool.rb | 48 ---------------------- 2 files changed, 33 insertions(+), 53 deletions(-) delete mode 100644 lib/gitlab/database/threaded_connection_pool.rb diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index 52e7f91bb01..f3863881229 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -6,24 +6,52 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration disable_ddl_transaction! def up - Gitlab::Database::ThreadedConnectionPool.with_pool(2) do |pool| - pool.execute_async <<-SQL.strip_heredoc + pool = create_connection_pool + threads = [] + + threads << Thread.new do + pool.with_connection do |connection| + connection.execute <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'BuildsEmailService' AND active IS FALSE AND properties = '{"notify_only_broken_builds":true}'; - SQL + SQL + end + end - pool.execute_async <<-SQL.strip_heredoc + threads << Thread.new do + pool.with_connection do |connection| + connection.execute <<-SQL.strip_heredoc DELETE FROM services WHERE type = 'PipelinesEmailService' AND active IS FALSE AND properties = '{"notify_only_broken_pipelines":true}'; - SQL + SQL + end end + + threads.each(&:join) end def down # Nothing can be done to restore the records end + + private + + def create_connection_pool + # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb + env = Rails.env + original_config = ActiveRecord::Base.configurations + env_config = original_config[env].merge('pool' => 2) + config = original_config.merge(env => env_config) + + spec = + ActiveRecord:: + ConnectionAdapters:: + ConnectionSpecification::Resolver.new(config).spec(env.to_sym) + + ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + end end diff --git a/lib/gitlab/database/threaded_connection_pool.rb b/lib/gitlab/database/threaded_connection_pool.rb deleted file mode 100644 index 1316b005741..00000000000 --- a/lib/gitlab/database/threaded_connection_pool.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Gitlab - module Database - class ThreadedConnectionPool - def self.with_pool(pool_size) - pool = new(pool_size) - - yield(pool) - - ensure - pool.join - pool.close - end - - def initialize(pool_size) - config = ActiveRecord::Base.configurations[Rails.env] - @ar_pool = ActiveRecord::Base.establish_connection( - config.merge(pool: pool_size)) - @workers = [] - @mutex = Mutex.new - end - - def execute_async(sql) - @mutex.synchronize do - @workers << Thread.new do - @ar_pool.with_connection do |connection| - connection.execute(sql) - end - end - end - end - - def join - threads = nil - - @mutex.synchronize do - threads = @workers.dup - @workers.clear - end - - threads.each(&:join) - end - - def close - @ar_pool.disconnect! - end - end - end -end -- cgit v1.2.1 From ae93d08b9cf01b8a12a5f6cacbd9c89e87e54ac4 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Sat, 11 Feb 2017 01:10:15 +0800 Subject: Disconnect the pool after done --- db/post_migrate/20170206040400_remove_inactive_default_email_services.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index f3863881229..249a9e805f1 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -32,6 +32,7 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration end threads.each(&:join) + pool.disconnect! end def down -- cgit v1.2.1 From ca659822257f21f09d5b22660ba7ae39395d79e6 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 14 Feb 2017 20:17:17 +0800 Subject: Use Gitlab::Database.with_connection_pool from !9192 --- ...40400_remove_inactive_default_email_services.rb | 63 ++++++++-------------- db/schema.rb | 2 +- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb index 249a9e805f1..a8e63e8bc7d 100644 --- a/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb +++ b/db/post_migrate/20170206040400_remove_inactive_default_email_services.rb @@ -6,53 +6,36 @@ class RemoveInactiveDefaultEmailServices < ActiveRecord::Migration disable_ddl_transaction! def up - pool = create_connection_pool - threads = [] - - threads << Thread.new do - pool.with_connection do |connection| - connection.execute <<-SQL.strip_heredoc - DELETE FROM services - WHERE type = 'BuildsEmailService' - AND active IS FALSE - AND properties = '{"notify_only_broken_builds":true}'; - SQL + Gitlab::Database.with_connection_pool(2) do |pool| + threads = [] + + threads << Thread.new do + pool.with_connection do |connection| + connection.execute <<-SQL.strip_heredoc + DELETE FROM services + WHERE type = 'BuildsEmailService' + AND active IS FALSE + AND properties = '{"notify_only_broken_builds":true}'; + SQL + end end - end - threads << Thread.new do - pool.with_connection do |connection| - connection.execute <<-SQL.strip_heredoc - DELETE FROM services - WHERE type = 'PipelinesEmailService' - AND active IS FALSE - AND properties = '{"notify_only_broken_pipelines":true}'; - SQL + threads << Thread.new do + pool.with_connection do |connection| + connection.execute <<-SQL.strip_heredoc + DELETE FROM services + WHERE type = 'PipelinesEmailService' + AND active IS FALSE + AND properties = '{"notify_only_broken_pipelines":true}'; + SQL + end end - end - threads.each(&:join) - pool.disconnect! + threads.each(&:join) + end end def down # Nothing can be done to restore the records end - - private - - def create_connection_pool - # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb - env = Rails.env - original_config = ActiveRecord::Base.configurations - env_config = original_config[env].merge('pool' => 2) - config = original_config.merge(env => env_config) - - spec = - ActiveRecord:: - ConnectionAdapters:: - ConnectionSpecification::Resolver.new(config).spec(env.to_sym) - - ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) - end end diff --git a/db/schema.rb b/db/schema.rb index de07b3837ad..d421d5c6774 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170204181513) do +ActiveRecord::Schema.define(version: 20170210075922) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" -- cgit v1.2.1