diff options
Diffstat (limited to 'lib')
12 files changed, 107 insertions, 32 deletions
diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml index 8063f3d1e69..97a9d94f42a 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6" CS_SCHEMA_MODEL: 15 container_scanning: diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml index 24c23ce89f3..9a4c75e7402 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6" CS_SCHEMA_MODEL: 15 container_scanning: diff --git a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb index 10603b3dbad..32d638380ea 100644 --- a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb +++ b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb @@ -33,6 +33,10 @@ module Gitlab 'NOT NULL' if query_result['not_null'] end + def partition_key? + query_result['partition_key'] + end + private attr_reader :query_result diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb index 30a13b5dff1..420195d89dd 100644 --- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb +++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb @@ -17,9 +17,10 @@ module Gitlab attr_reader :table_name - def initialize(table_name, pg_query_stmt) + def initialize(table_name, pg_query_stmt, partitioning_stmt) @table_name = table_name @pg_query_stmt = pg_query_stmt + @partitioning_stmt = partitioning_stmt end def name @@ -44,9 +45,13 @@ module Gitlab 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR } end + def partition_key? + partition_keys.include?(name) + end + private - attr_reader :pg_query_stmt + attr_reader :pg_query_stmt, :partitioning_stmt def constraints @constraints ||= pg_query_stmt.constraints @@ -81,6 +86,7 @@ module Gitlab # Parses PGQuery nodes recursively # # :constraint:: nodes that groups column default info + # :partition_elem:: node that store partition key info # :func_cal:: nodes that stores functions, like +now()+ # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+ # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+ @@ -93,6 +99,8 @@ module Gitlab case node.node when :constraint parse_node(node.constraint.raw_expr) + when :partition_elem + node.partition_elem.name when :func_call "#{parse_node(node.func_call.funcname.first)}()" when :a_const @@ -107,6 +115,12 @@ module Gitlab node.to_h[node.node].values.last end end + + def partition_keys + return [] unless partitioning_stmt + + @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) } + end end end end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb index 9ff4a843e6d..7a0e348a27b 100644 --- a/lib/gitlab/database/schema_validation/database.rb +++ b/lib/gitlab/database/schema_validation/database.rb @@ -105,12 +105,14 @@ module Gitlab table_information.relname AS table_name, col_information.attname AS column_name, col_information.attnotnull AS not_null, + col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key, format_type(col_information.atttypid, col_information.atttypmod) AS data_type, pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default FROM pg_attribute AS col_information JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid - LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid + LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid + LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid AND col_information.attnum = col_default_information.adnum WHERE NOT col_information.attisdropped AND col_information.attnum > 0 diff --git a/lib/gitlab/database/schema_validation/inconsistency_filter.rb b/lib/gitlab/database/schema_validation/inconsistency_filter.rb new file mode 100644 index 00000000000..aa3a71c0edb --- /dev/null +++ b/lib/gitlab/database/schema_validation/inconsistency_filter.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class InconsistencyFilter + def initialize(tables, triggers) + @tables = tables + @triggers = triggers + end + + def to_proc + proc do |inconsistency| + inconsistency unless ignored?(inconsistency) + end + end + + private + + attr_reader :tables, :triggers + + def ignored?(inconsistency) + case inconsistency.type + in 'extra_tables' | 'missing_tables' + ignored_table?(inconsistency.table_name) + in 'extra_triggers' | 'missing_triggers' + ignored_trigger?(inconsistency.object_name) + else + false + end + end + + def ignored_table?(name) + tables.include?(name) + end + + def ignored_trigger?(name) + triggers.any? { |ignored_object| name.to_s.include?(ignored_object) } + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/column.rb b/lib/gitlab/database/schema_validation/schema_objects/column.rb index 38ad8e309a3..bd219300a13 100644 --- a/lib/gitlab/database/schema_validation/schema_objects/column.rb +++ b/lib/gitlab/database/schema_validation/schema_objects/column.rb @@ -11,7 +11,7 @@ module Gitlab attr_reader :adapter - delegate :name, :table_name, to: :adapter + delegate :name, :table_name, :partition_key?, to: :adapter def statement [name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ') diff --git a/lib/gitlab/database/schema_validation/schema_objects/table.rb b/lib/gitlab/database/schema_validation/schema_objects/table.rb index 6f573e7027f..de2e675d20e 100644 --- a/lib/gitlab/database/schema_validation/schema_objects/table.rb +++ b/lib/gitlab/database/schema_validation/schema_objects/table.rb @@ -31,7 +31,7 @@ module Gitlab private def columns_statement - columns.map(&:statement).join(', ') + columns.reject(&:partition_key?).map(&:statement).join(', ') end end end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb index e93c33aedcd..299db1c2a7e 100644 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -38,9 +38,11 @@ module Gitlab def tables @tables ||= table_statements.map do |stmt| table_name = stmt.relation.relname + partition_stmt = stmt.partspec columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column| - SchemaObjects::Column.new(Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def)) + adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt) + SchemaObjects::Column.new(adapter) end SchemaObjects::Table.new(table_name, columns) diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index deef3a9a38b..52aee4d2d45 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -9,11 +9,15 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists action = action.to_s + + project_id = project.is_a?(Integer) ? project : project&.id + contexts = [ Tracking::StandardContext.new( - project: project, - user: user, - namespace: namespace, + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name, + project_id: project_id, + user_id: user&.id, **extra).to_context, *context ] @@ -25,9 +29,10 @@ module Gitlab destination = Gitlab::Tracking::Destinations::DatabaseEventsSnowplow.new contexts = [ Tracking::StandardContext.new( - project: project, - user: user, - namespace: namespace, + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name, + project_id: project&.id, + user_id: user&.id, **extra).to_context, *context ] diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 50467de44b8..62c45368410 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -6,15 +6,16 @@ module Gitlab GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' GITLAB_RAILS_SOURCE = 'gitlab-rails' - def initialize(namespace: nil, project: nil, user: nil, **extra) - check_argument_type(:namespace, namespace, [Namespace]) - check_argument_type(:project, project, [Project, Integer]) - check_argument_type(:user, user, [User, DeployToken]) - - @namespace = namespace - @plan = namespace&.actual_plan_name - @project = project - @user = user + def initialize(namespace_id: nil, plan_name: nil, project_id: nil, user_id: nil, **extra) + check_argument_type(:namespace_id, namespace_id, [Integer]) + check_argument_type(:plan_name, plan_name, [String]) + check_argument_type(:project_id, project_id, [Integer]) + check_argument_type(:user_id, user_id, [Integer]) + + @namespace_id = namespace_id + @plan_name = plan_name + @project_id = project_id + @user_id = user_id @extra = extra end @@ -40,25 +41,21 @@ module Gitlab private - attr_accessor :namespace, :project, :extra, :plan, :user + attr_accessor :namespace_id, :project_id, :extra, :plan_name, :user_id def to_h { environment: environment, source: source, - plan: plan, + plan: plan_name, extra: extra, - user_id: user&.id, - namespace_id: namespace&.id, + user_id: user_id, + namespace_id: namespace_id, project_id: project_id, context_generated_at: Time.current } end - def project_id - project.is_a?(Integer) ? project : project&.id - end - def check_argument_type(argument_name, argument_value, allowed_classes) return if argument_value.nil? || allowed_classes.any? { |allowed_class| argument_value.is_a?(allowed_class) } diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 372ebcfbee8..ad5595afe0e 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -449,6 +449,11 @@ namespace :gitlab do end namespace :schema_checker do + # TODO: Remove `test_replication` after PG 14 upgrade is finished + # https://gitlab.com/gitlab-com/gl-infra/db-migration/-/merge_requests/406#note_1369214728 + IGNORED_TABLES = %w[test_replication].freeze + IGNORED_TRIGGERS = ['gitlab_schema_write_trigger_for_'].freeze + desc 'Checks schema inconsistencies' task run: :environment do database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] @@ -457,7 +462,10 @@ namespace :gitlab do stucture_sql_path = Rails.root.join('db/structure.sql') structure_sql = Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) - inconsistencies = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database).execute + filter = Gitlab::Database::SchemaValidation::InconsistencyFilter.new(IGNORED_TABLES, IGNORED_TRIGGERS) + + inconsistencies = + Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database).execute.filter_map(&filter) gitlab_url = 'gitlab-org/gitlab' |