summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--GITLAB_SHELL_VERSION2
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.lock4
-rw-r--r--app/models/application_setting.rb3
-rw-r--r--app/models/application_setting_implementation.rb9
-rw-r--r--app/services/application_settings/update_service.rb2
-rw-r--r--changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml5
-rw-r--r--changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml5
-rw-r--r--changelogs/unreleased/sh-bump-gitlab-shell-version.yml5
-rw-r--r--changelogs/unreleased/sh-fix-kroki-format-filtering.yml5
-rw-r--r--db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb21
-rw-r--r--db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb21
-rw-r--r--db/schema_migrations/202103012006011
-rw-r--r--db/schema_migrations/202103012009591
-rw-r--r--db/structure.sql2
-rw-r--r--doc/administration/object_storage.md18
-rw-r--r--lib/gitlab/background_migration/backfill_artifact_expiry_date.rb13
-rw-r--r--lib/gitlab/kroki.rb3
-rw-r--r--spec/deprecation_toolkit_env.rb1
-rw-r--r--spec/lib/gitlab/kroki_spec.rb38
-rw-r--r--spec/models/application_setting_spec.rb26
-rw-r--r--spec/services/application_settings/update_service_spec.rb1
-rwxr-xr-x[-rw-r--r--]vendor/gitignore/C++.gitignore0
-rwxr-xr-x[-rw-r--r--]vendor/gitignore/Java.gitignore0
24 files changed, 169 insertions, 19 deletions
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION
index 592a1a89678..15ede847472 100644
--- a/GITLAB_SHELL_VERSION
+++ b/GITLAB_SHELL_VERSION
@@ -1 +1 @@
-13.16.1
+13.17.0
diff --git a/Gemfile b/Gemfile
index 0445cc391aa..ec253c37b3a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -156,7 +156,7 @@ gem 'wikicloth', '0.8.1'
gem 'asciidoctor', '~> 2.0.10'
gem 'asciidoctor-include-ext', '~> 0.3.1', require: false
gem 'asciidoctor-plantuml', '~> 0.0.12'
-gem 'asciidoctor-kroki', '~> 0.3.0', require: false
+gem 'asciidoctor-kroki', '~> 0.4.0', require: false
gem 'rouge', '~> 3.26.0'
gem 'truncato', '~> 0.7.11'
gem 'bootstrap_form', '~> 4.2.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 90ee2eaec09..1b6701e8e3d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -84,7 +84,7 @@ GEM
asciidoctor (2.0.12)
asciidoctor-include-ext (0.3.1)
asciidoctor (>= 1.5.6, < 3.0.0)
- asciidoctor-kroki (0.3.0)
+ asciidoctor-kroki (0.4.0)
asciidoctor (~> 2.0)
asciidoctor-plantuml (0.0.12)
asciidoctor (>= 1.5.6, < 3.0.0)
@@ -1291,7 +1291,7 @@ DEPENDENCIES
asana (~> 0.10.3)
asciidoctor (~> 2.0.10)
asciidoctor-include-ext (~> 0.3.1)
- asciidoctor-kroki (~> 0.3.0)
+ asciidoctor-kroki (~> 0.4.0)
asciidoctor-plantuml (~> 0.0.12)
atlassian-jwt (~> 0.2.0)
attr_encrypted (~> 3.1.0)
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 33c058dab96..4959401eb27 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -58,8 +58,9 @@ class ApplicationSetting < ApplicationRecord
serialize :domain_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :domain_denylist, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize
- serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
+
# See https://gitlab.com/gitlab-org/gitlab/-/issues/300916
+ serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :asset_proxy_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize
cache_markdown_field :sign_in_text
diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index 2911ae6b1c8..08c16930b13 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -280,13 +280,18 @@ module ApplicationSettingImplementation
self.notes_create_limit_allowlist = strings_to_array(values).map(&:downcase)
end
- def asset_proxy_allowlist=(values)
+ def asset_proxy_whitelist=(values)
values = strings_to_array(values) if values.is_a?(String)
# make sure we always allow the running host
values << Gitlab.config.gitlab.host unless values.include?(Gitlab.config.gitlab.host)
- self[:asset_proxy_allowlist] = values
+ self[:asset_proxy_whitelist] = values
+ end
+ alias_method :asset_proxy_allowlist=, :asset_proxy_whitelist=
+
+ def asset_proxy_allowlist
+ read_attribute(:asset_proxy_whitelist)
end
def repository_storages
diff --git a/app/services/application_settings/update_service.rb b/app/services/application_settings/update_service.rb
index 5e5c8ae2177..7792b811b4e 100644
--- a/app/services/application_settings/update_service.rb
+++ b/app/services/application_settings/update_service.rb
@@ -6,7 +6,7 @@ module ApplicationSettings
attr_reader :params, :application_setting
- MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_allowlist).freeze
+ MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_whitelist).freeze
def execute
result = update_settings
diff --git a/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml b/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml
new file mode 100644
index 00000000000..e5c9eeae035
--- /dev/null
+++ b/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml
@@ -0,0 +1,5 @@
+---
+title: Update Kroki to fix Wavedrom graphs
+merge_request: 55659
+author:
+type: fixed
diff --git a/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml b/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml
new file mode 100644
index 00000000000..36d3730448a
--- /dev/null
+++ b/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml
@@ -0,0 +1,5 @@
+---
+title: Rename asset_proxy_allowlist column
+merge_request: 55884
+author:
+type: fixed
diff --git a/changelogs/unreleased/sh-bump-gitlab-shell-version.yml b/changelogs/unreleased/sh-bump-gitlab-shell-version.yml
new file mode 100644
index 00000000000..fef8885c0a8
--- /dev/null
+++ b/changelogs/unreleased/sh-bump-gitlab-shell-version.yml
@@ -0,0 +1,5 @@
+---
+title: Upgrade gitlab-shell to v13.17.0
+merge_request: 55295
+author:
+type: fixed
diff --git a/changelogs/unreleased/sh-fix-kroki-format-filtering.yml b/changelogs/unreleased/sh-fix-kroki-format-filtering.yml
new file mode 100644
index 00000000000..7c5f3d210ae
--- /dev/null
+++ b/changelogs/unreleased/sh-fix-kroki-format-filtering.yml
@@ -0,0 +1,5 @@
+---
+title: Fix disabling of Kroki optional formats
+merge_request: 55665
+author:
+type: fixed
diff --git a/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb b/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb
new file mode 100644
index 00000000000..8a9acd8b86e
--- /dev/null
+++ b/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class RenameAssetProxyAllowlistOnApplicationSettings < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers::V2
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ rename_column_concurrently :application_settings,
+ :asset_proxy_allowlist,
+ :asset_proxy_whitelist
+ end
+
+ def down
+ undo_rename_column_concurrently :application_settings,
+ :asset_proxy_allowlist,
+ :asset_proxy_whitelist
+ end
+end
diff --git a/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb b/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb
new file mode 100644
index 00000000000..d0b372b84f0
--- /dev/null
+++ b/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CleanUpAssetProxyAllowlistRenameOnApplicationSettings < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers::V2
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ cleanup_concurrent_column_rename :application_settings,
+ :asset_proxy_allowlist,
+ :asset_proxy_whitelist
+ end
+
+ def down
+ undo_cleanup_concurrent_column_rename :application_settings,
+ :asset_proxy_allowlist,
+ :asset_proxy_whitelist
+ end
+end
diff --git a/db/schema_migrations/20210301200601 b/db/schema_migrations/20210301200601
new file mode 100644
index 00000000000..13907f9de8b
--- /dev/null
+++ b/db/schema_migrations/20210301200601
@@ -0,0 +1 @@
+21ae7ea7cbf1d34c7b9dc300a641eaf975ed1e33f5bc519494cd37c4a661bec8 \ No newline at end of file
diff --git a/db/schema_migrations/20210301200959 b/db/schema_migrations/20210301200959
new file mode 100644
index 00000000000..8c5efabd76d
--- /dev/null
+++ b/db/schema_migrations/20210301200959
@@ -0,0 +1 @@
+28b90c9b7c2e4f2e2b12088f5aee85c16dfb567f89ed6a8e771f2c5d91c818d9 \ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 97a2850fd5c..c6b3b7ea997 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -9394,11 +9394,11 @@ CREATE TABLE application_settings (
container_registry_cleanup_tags_service_max_list_size integer DEFAULT 200 NOT NULL,
enforce_ssh_key_expiration boolean DEFAULT false NOT NULL,
git_two_factor_session_expiry integer DEFAULT 15 NOT NULL,
- asset_proxy_allowlist text,
keep_latest_artifact boolean DEFAULT true NOT NULL,
notes_create_limit integer DEFAULT 300 NOT NULL,
notes_create_limit_allowlist text[] DEFAULT '{}'::text[] NOT NULL,
kroki_formats jsonb DEFAULT '{}'::jsonb NOT NULL,
+ asset_proxy_whitelist text,
CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),
diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md
index 3cad18dc497..3b43f53dc82 100644
--- a/doc/administration/object_storage.md
+++ b/doc/administration/object_storage.md
@@ -31,6 +31,8 @@ GitLab has been tested on a number of object storage providers:
HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806).
Be sure to upgrade to GitLab v13.3.0 or above if you use S3 storage with this hardware.
+- Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying).
+
## Configuration guides
There are two ways of specifying object storage configuration in GitLab:
@@ -752,7 +754,6 @@ To set up an instance profile:
"Action": [
"s3:PutObject",
"s3:GetObject",
- "s3:AbortMultipartUpload",
"s3:DeleteObject"
],
"Resource": "arn:aws:s3:::test-bucket/*"
@@ -764,3 +765,18 @@ To set up an instance profile:
1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/)
to the EC2 instance hosting your GitLab instance.
1. Configure GitLab to use it via the `use_iam_profile` configuration option.
+
+### Multi-threaded copying
+
+GitLab uses the [S3 Upload Part Copy API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html)
+to accelerate the copying of files within a bucket. Ceph S3 [prior to Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/)
+does not support this and [returns a 404 error when files are copied during the upload process](https://gitlab.com/gitlab-org/gitlab/-/issues/300604).
+
+The feature can be disabled using the `:s3_multithreaded_uploads`
+feature flag. To disable the feature, ask a GitLab administrator with
+[Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags)
+to run the following command:
+
+```ruby
+Feature.disable(:s3_multithreaded_uploads)
+```
diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb
index 0a8c203421b..f6b36571c90 100644
--- a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb
+++ b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb
@@ -6,9 +6,9 @@ module Gitlab
class BackfillArtifactExpiryDate
include Gitlab::Utils::StrongMemoize
- BATCH_SIZE = 1_000
- DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze
+ SWITCH_DATE = Date.new(2020, 06, 22).freeze
OLD_ARTIFACT_AGE = 15.months
+ BATCH_SIZE = 1_000
OLD_ARTIFACT_EXPIRY_OFFSET = 3.months
RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year
@@ -18,16 +18,17 @@ module Gitlab
self.table_name = 'ci_job_artifacts'
- scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
- scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) }
scope :without_expiry_date, -> { where(expire_at: nil) }
+ scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) }
+ scope :between, -> (start_id, end_id) { where(id: start_id..end_id) }
scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) }
scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) }
end
def perform(start_id, end_id)
- Ci::JobArtifact.between(start_id, end_id)
- .without_expiry_date.before_default_expiration_switch
+ Ci::JobArtifact
+ .without_expiry_date.before_switch
+ .between(start_id, end_id)
.each_batch(of: BATCH_SIZE) do |batch|
batch.old.update_all(expire_at: old_artifact_expiry_date)
batch.recent.update_all(expire_at: recent_artifact_expiry_date)
diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb
index 2948b6ebd5b..fa10e922c80 100644
--- a/lib/gitlab/kroki.rb
+++ b/lib/gitlab/kroki.rb
@@ -31,9 +31,6 @@ module Gitlab
DIAGRAMS_FORMATS
end
- # No additional diagram formats
- return diagram_formats unless current_settings.kroki_formats.present?
-
# Diagrams that require a companion container must be explicitly enabled from the settings
diagram_formats.select do |diagram_type|
current_settings.kroki_format_supported?(diagram_type)
diff --git a/spec/deprecation_toolkit_env.rb b/spec/deprecation_toolkit_env.rb
index d2ff2d2cb37..10eaaf13aaa 100644
--- a/spec/deprecation_toolkit_env.rb
+++ b/spec/deprecation_toolkit_env.rb
@@ -61,6 +61,7 @@ module DeprecationToolkitEnv
batch-loader-1.4.0/lib/batch_loader/graphql.rb
carrierwave-1.3.1/lib/carrierwave/sanitized_file.rb
activerecord-6.0.3.4/lib/active_record/relation.rb
+ asciidoctor-2.0.12/lib/asciidoctor/extensions.rb
]
end
diff --git a/spec/lib/gitlab/kroki_spec.rb b/spec/lib/gitlab/kroki_spec.rb
new file mode 100644
index 00000000000..31d3edd158b
--- /dev/null
+++ b/spec/lib/gitlab/kroki_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Gitlab::Kroki do
+ using RSpec::Parameterized::TableSyntax
+
+ describe '.formats' do
+ def default_formats
+ %w[bytefield c4plantuml ditaa erd graphviz nomnoml plantuml svgbob umlet vega vegalite wavedrom].freeze
+ end
+
+ subject { described_class.formats(Gitlab::CurrentSettings) }
+
+ where(:enabled_formats, :expected_formats) do
+ '' | default_formats
+ 'blockdiag' | default_formats + %w[actdiag blockdiag nwdiag packetdiag rackdiag seqdiag]
+ 'bpmn' | default_formats + %w[bpmn]
+ 'excalidraw' | default_formats + %w[excalidraw]
+ end
+
+ with_them do
+ before do
+ kroki_formats =
+ if enabled_formats.present?
+ { enabled_formats => true }
+ else
+ {}
+ end
+
+ stub_application_setting(kroki_enabled: true, kroki_url: "http://localhost:8000", kroki_formats: kroki_formats)
+ end
+
+ it 'returns the expected formats' do
+ expect(subject).to match_array(expected_formats)
+ end
+ end
+ end
+end
diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb
index 9a4dd2c799b..5658057f588 100644
--- a/spec/models/application_setting_spec.rb
+++ b/spec/models/application_setting_spec.rb
@@ -650,6 +650,32 @@ RSpec.describe ApplicationSetting do
end
end
+ describe '#asset_proxy_whitelist' do
+ context 'when given an Array' do
+ it 'sets the domains and adds current running host' do
+ setting.asset_proxy_whitelist = ['example.com', 'assets.example.com']
+ expect(setting.asset_proxy_whitelist).to eq(['example.com', 'assets.example.com', 'localhost'])
+ end
+ end
+
+ context 'when given a String' do
+ it 'sets multiple domains with spaces' do
+ setting.asset_proxy_whitelist = 'example.com *.example.com'
+ expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+ end
+
+ it 'sets multiple domains with newlines and a space' do
+ setting.asset_proxy_whitelist = "example.com\n *.example.com"
+ expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+ end
+
+ it 'sets multiple domains with commas' do
+ setting.asset_proxy_whitelist = "example.com, *.example.com"
+ expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost'])
+ end
+ end
+ end
+
describe '#asset_proxy_allowlist' do
context 'when given an Array' do
it 'sets the domains and adds current running host' do
diff --git a/spec/services/application_settings/update_service_spec.rb b/spec/services/application_settings/update_service_spec.rb
index 1352a595ba4..258b3d25aee 100644
--- a/spec/services/application_settings/update_service_spec.rb
+++ b/spec/services/application_settings/update_service_spec.rb
@@ -123,6 +123,7 @@ RSpec.describe ApplicationSettings::UpdateService do
it_behaves_like 'invalidates markdown cache', { asset_proxy_url: 'http://test.com' }
it_behaves_like 'invalidates markdown cache', { asset_proxy_secret_key: 'another secret' }
it_behaves_like 'invalidates markdown cache', { asset_proxy_allowlist: ['domain.com'] }
+ it_behaves_like 'invalidates markdown cache', { asset_proxy_whitelist: ['domain.com'] }
context 'when also setting the local_markdown_version' do
let(:params) { { asset_proxy_enabled: true, local_markdown_version: 12 } }
diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore
index 259148fa18f..259148fa18f 100644..100755
--- a/vendor/gitignore/C++.gitignore
+++ b/vendor/gitignore/C++.gitignore
diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore
index a1c2a238a96..a1c2a238a96 100644..100755
--- a/vendor/gitignore/Java.gitignore
+++ b/vendor/gitignore/Java.gitignore