summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--changelogs/unreleased/246741-conan-unicorn-bug.yml5
-rw-r--r--changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml6
-rw-r--r--db/migrate/20200515152649_enable_btree_gist_extension.rb6
-rw-r--r--doc/install/postgresql_extensions.md76
-rw-r--r--doc/install/requirements.md7
-rw-r--r--doc/update/README.md1
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb2
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/database/migration_helpers.rb57
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb6
-rw-r--r--locale/gitlab.pot2
-rw-r--r--spec/lib/gitlab/database/migration_helpers_spec.rb52
-rw-r--r--spec/requests/api/conan_packages_spec.rb12
14 files changed, 223 insertions, 13 deletions
diff --git a/changelogs/unreleased/246741-conan-unicorn-bug.yml b/changelogs/unreleased/246741-conan-unicorn-bug.yml
new file mode 100644
index 00000000000..669550c1c81
--- /dev/null
+++ b/changelogs/unreleased/246741-conan-unicorn-bug.yml
@@ -0,0 +1,5 @@
+---
+title: Use 'read' method to get request body in Conan to fix uploads when using Unicorn
+merge_request: 41801
+author:
+type: fixed
diff --git a/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml b/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml
new file mode 100644
index 00000000000..6000a9f493d
--- /dev/null
+++ b/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml
@@ -0,0 +1,6 @@
+---
+title: Fixes Auto DevOps deploy script for multiple additional hosts separated by
+ comma and space
+merge_request: 41404
+author:
+type: fixed
diff --git a/db/migrate/20200515152649_enable_btree_gist_extension.rb b/db/migrate/20200515152649_enable_btree_gist_extension.rb
index 686b685fb5d..bd81c921a87 100644
--- a/db/migrate/20200515152649_enable_btree_gist_extension.rb
+++ b/db/migrate/20200515152649_enable_btree_gist_extension.rb
@@ -1,13 +1,15 @@
# frozen_string_literal: true
class EnableBtreeGistExtension < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
DOWNTIME = false
def up
- execute 'CREATE EXTENSION IF NOT EXISTS btree_gist'
+ create_extension :btree_gist
end
def down
- execute 'DROP EXTENSION IF EXISTS btree_gist'
+ drop_extension :btree_gist
end
end
diff --git a/doc/install/postgresql_extensions.md b/doc/install/postgresql_extensions.md
new file mode 100644
index 00000000000..4156d72097d
--- /dev/null
+++ b/doc/install/postgresql_extensions.md
@@ -0,0 +1,76 @@
+---
+last_updated: 2020-09-01
+---
+
+# Managing PostgreSQL extensions
+
+This guide documents how to manage PostgreSQL extensions for installations with an external
+PostgreSQL database.
+
+GitLab requires certain extensions to be installed into the GitLab database. For example,
+GitLab relies on `pg_trgm` and the `btree_gist` extensions.
+
+In order to install extensions, PostgreSQL requires the user to have superuser privileges.
+Typically, the GitLab database user is not a superuser. Therefore, regular database migrations
+cannot be used in installing extensions and instead, extensions have to be installed manually
+prior to upgrading GitLab to a newer version.
+
+## Installing PostgreSQL extensions manually
+
+In order to install a PostgreSQL extension, this procedure should be followed:
+
+1. Connect to the GitLab PostgreSQL database using a superuser, for example:
+
+ ```shell
+ sudo gitlab-psql -d gitlabhq_production
+ ```
+
+1. Install the extension (`btree_gist` in this example) using [`CREATE EXTENSION`](https://www.postgresql.org/docs/11/sql-createextension.html):
+
+ ```sql
+ CREATE EXTENSION IF NOT EXISTS btree_gist
+ ```
+
+1. Verify installed extensions:
+
+ ```shell
+ gitlabhq_production=# \dx
+ List of installed extensions
+ Name | Version | Schema | Description
+ ------------+---------+------------+-------------------------------------------------------------------
+ btree_gist | 1.5 | public | support for indexing common datatypes in GiST
+ pg_trgm | 1.4 | public | text similarity measurement and index searching based on trigrams
+ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language
+ (3 rows)
+ ```
+
+On some systems you may need to install an additional package (for example,
+`postgresql-contrib`) for certain extensions to become available.
+
+## A typical migration failure scenario
+
+The following is an example of a situation when the extension hasn't been installed before running migrations.
+In this scenario, the database migration fails to create the extension `btree_gist` because of insufficient
+privileges.
+
+```shell
+== 20200515152649 EnableBtreeGistExtension: migrating =========================
+-- execute("CREATE EXTENSION IF NOT EXISTS btree_gist")
+
+GitLab requires the PostgreSQL extension 'btree_gist' installed in database 'gitlabhq_production', but
+the database user is not allowed to install the extension.
+
+You can either install the extension manually using a database superuser:
+
+ CREATE EXTENSION IF NOT EXISTS btree_gist
+
+Or, you can solve this by logging in to the GitLab database (gitlabhq_production) using a superuser and running:
+
+ ALTER regular WITH SUPERUSER
+
+This query will grant the user superuser permissions, ensuring any database extensions
+can be installed through migrations.
+```
+
+In order to recover from this situation, the extension needs to be installed manually using a superuser, and
+the database migration (or GitLab upgrade) can be retried afterwards.
diff --git a/doc/install/requirements.md b/doc/install/requirements.md
index 4d9f09bb8b0..10d5853c82e 100644
--- a/doc/install/requirements.md
+++ b/doc/install/requirements.md
@@ -140,11 +140,8 @@ GitLab version | Minimum PostgreSQL version
12.10 | 11
13.0 | 11
-You must also ensure the `pg_trgm` and `btree_gist` extensions are loaded into every
-GitLab database. These extensions [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) using a PostgreSQL super user.
-
-On some systems you may need to install an additional package (for example,
-`postgresql-contrib`) for this extension to become available.
+You must also ensure the `pg_trgm` and `btree_gist` extensions are [loaded into every
+GitLab database](postgresql_extensions.html).
NOTE: **Note:**
Support for [PostgreSQL 9.6 and 10 has been removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184).
diff --git a/doc/update/README.md b/doc/update/README.md
index 85fc4363673..a7f7aaf5887 100644
--- a/doc/update/README.md
+++ b/doc/update/README.md
@@ -310,3 +310,4 @@ for more information.
- [Restoring from backup after a failed upgrade](restore_after_failure.md)
- [Upgrading PostgreSQL Using Slony](upgrading_postgresql_using_slony.md), for
upgrading a PostgreSQL database with minimal downtime.
+- [Managing PostgreSQL extensions](../install/postgresql_extensions.md)
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index c9c2f66ef62..1161d1386bb 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -139,7 +139,7 @@ module API
end
def file_names
- json_payload = Gitlab::Json.parse(request.body.string)
+ json_payload = Gitlab::Json.parse(request.body.read)
bad_request!(nil) unless json_payload.is_a?(Hash)
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index e9d77766db3..86f946aafda 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.2"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
dast_environment_deploy:
extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 41120750ff4..2922e1c6e88 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.2"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
dependencies: []
include:
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index a618a3017b2..b62b6e20dd5 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1212,6 +1212,63 @@ into similar problems in the future (e.g. when new tables are created).
)
end
+ def create_extension(extension)
+ execute('CREATE EXTENSION IF NOT EXISTS %s' % extension)
+ rescue ActiveRecord::StatementInvalid => e
+ dbname = Database.database_name
+ user = Database.username
+
+ warn(<<~MSG) if e.to_s =~ /permission denied/
+ GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but
+ the database user is not allowed to install the extension.
+
+ You can either install the extension manually using a database superuser:
+
+ CREATE EXTENSION IF NOT EXISTS #{extension}
+
+ Or, you can solve this by logging in to the GitLab
+ database (#{dbname}) using a superuser and running:
+
+ ALTER #{user} WITH SUPERUSER
+
+ This query will grant the user superuser permissions, ensuring any database extensions
+ can be installed through migrations.
+
+ For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
+ MSG
+
+ raise
+ end
+
+ def drop_extension(extension)
+ execute('DROP EXTENSION IF EXISTS %s' % extension)
+ rescue ActiveRecord::StatementInvalid => e
+ dbname = Database.database_name
+ user = Database.username
+
+ warn(<<~MSG) if e.to_s =~ /permission denied/
+ This migration attempts to drop the PostgreSQL extension '#{extension}'
+ installed in database '#{dbname}', but the database user is not allowed
+ to drop the extension.
+
+ You can either drop the extension manually using a database superuser:
+
+ DROP EXTENSION IF EXISTS #{extension}
+
+ Or, you can solve this by logging in to the GitLab
+ database (#{dbname}) using a superuser and running:
+
+ ALTER #{user} WITH SUPERUSER
+
+ This query will grant the user superuser permissions, ensuring any database extensions
+ can be dropped through migrations.
+
+ For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
+ MSG
+
+ raise
+ end
+
private
def validate_check_constraint_name!(constraint_name)
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index e6d8ec55319..84b6fb9f76e 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -55,8 +55,10 @@ module Gitlab
partitioned_table_name = make_partitioned_table_name(table_name)
- create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
- create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
+ transaction do
+ create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
+ create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
+ end
create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
end
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index ed9d4b43f5a..e81e3f3b2e0 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -28682,7 +28682,7 @@ msgstr ""
msgid "ciReport|%{linkStartTag}Learn more about SAST %{linkEndTag}"
msgstr ""
-msgid "ciReport|%{linkStartTag}Learn more about Secret Scanning %{linkEndTag}"
+msgid "ciReport|%{linkStartTag}Learn more about Secret Detection %{linkEndTag}"
msgstr ""
msgid "ciReport|%{linkStartTag}Learn more about codequality reports %{linkEndTag}"
diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb
index 4b7f371b25a..7d26fbb1132 100644
--- a/spec/lib/gitlab/database/migration_helpers_spec.rb
+++ b/spec/lib/gitlab/database/migration_helpers_spec.rb
@@ -2329,4 +2329,56 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
end
end
end
+
+ describe '#create_extension' do
+ subject { model.create_extension(extension) }
+
+ let(:extension) { :btree_gist }
+
+ it 'executes CREATE EXTENSION statement' do
+ expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
+
+ subject
+ end
+
+ context 'without proper permissions' do
+ before do
+ allow(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
+ end
+
+ it 'raises the exception' do
+ expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
+ end
+
+ it 'prints an error message' do
+ expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error
+ end
+ end
+ end
+
+ describe '#drop_extension' do
+ subject { model.drop_extension(extension) }
+
+ let(:extension) { 'btree_gist' }
+
+ it 'executes CREATE EXTENSION statement' do
+ expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/)
+
+ subject
+ end
+
+ context 'without proper permissions' do
+ before do
+ allow(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
+ end
+
+ it 'raises the exception' do
+ expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
+ end
+
+ it 'prints an error message' do
+ expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error
+ end
+ end
+ end
end
diff --git a/spec/requests/api/conan_packages_spec.rb b/spec/requests/api/conan_packages_spec.rb
index 7a97743ede1..d24f6b68048 100644
--- a/spec/requests/api/conan_packages_spec.rb
+++ b/spec/requests/api/conan_packages_spec.rb
@@ -363,6 +363,16 @@ RSpec.describe API::ConanPackages do
end
end
+ shared_examples 'successful response when using Unicorn' do
+ context 'on Unicorn', :unicorn do
+ it 'returns successfully' do
+ subject
+
+ expect(response).to have_gitlab_http_status(:ok)
+ end
+ end
+ end
+
describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do
let(:recipe_path) { package.conan_recipe_path }
@@ -458,6 +468,7 @@ RSpec.describe API::ConanPackages do
it_behaves_like 'rejects invalid recipe'
it_behaves_like 'rejects invalid upload_url params'
+ it_behaves_like 'successful response when using Unicorn'
it 'returns a set of upload urls for the files requested' do
subject
@@ -521,6 +532,7 @@ RSpec.describe API::ConanPackages do
it_behaves_like 'rejects invalid recipe'
it_behaves_like 'rejects invalid upload_url params'
+ it_behaves_like 'successful response when using Unicorn'
it 'returns a set of upload urls for the files requested' do
expected_response = {
View Lorry Controller Status