diff options
104 files changed, 2790 insertions, 495 deletions
diff --git a/.haml-lint.yml b/.haml-lint.yml index bad918ef35d..e9cc4a91a21 100644 --- a/.haml-lint.yml +++ b/.haml-lint.yml @@ -97,6 +97,7 @@ linters: - Cop/LineBreakAroundConditionalBlock - Cop/ProjectPathHelper - GitlabSecurity/PublicSend + - Layout/EmptyLineAfterGuardClause - Layout/LeadingCommentSpace - Layout/SpaceAfterColon - Layout/SpaceAfterComma @@ -112,11 +113,13 @@ linters: - Lint/LiteralInInterpolation - Lint/ParenthesesAsGroupedExpression - Lint/RedundantWithIndex + - Lint/SafeNavigationConsistency - Lint/Syntax - Metrics/BlockNesting - Naming/VariableName - Performance/RedundantMatch - Performance/StringReplacement + - Rails/LinkToBlank - Rails/Presence - Rails/RequestReferer - Style/AndOr @@ -134,6 +137,7 @@ linters: - Style/TernaryParentheses - Style/TrailingCommaInHashLiteral - Style/UnlessElse + - Style/UnneededCondition - Style/WordArray - Style/ZeroLengthPredicate diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 77ad4753c84..63b1685feda 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,42 +1,154 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2018-01-18 18:23:26 +0100 using RuboCop version 0.52.1. +# on 2019-05-04 16:01:00 +0000 using RuboCop version 0.68.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 181 +# Offense count: 264 Capybara/CurrentPathExpectation: Enabled: false -# Offense count: 167 +# Offense count: 1097 # Cop supports --auto-correct. -Layout/EmptyLinesAroundArguments: +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: with_first_argument, with_fixed_indentation +Layout/AlignArguments: Enabled: false -# Offense count: 83 +# Offense count: 824 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle. +# SupportedHashRocketStyles: key, separator, table +# SupportedColonStyles: key, separator, table +# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit +Layout/AlignHash: + Enabled: false + +# Offense count: 13 +# Cop supports --auto-correct. +Layout/ClosingHeredocIndentation: + Exclude: + - 'app/graphql/mutations/merge_requests/set_wip.rb' + - 'ee/db/geo/migrate/20180322062741_migrate_ci_job_artifacts_to_separate_registry.rb' + - 'ee/db/migrate/20160204190809_update_jenkins_service_category.rb' + - 'ee/lib/gitlab/background_migration/prune_orphaned_geo_events.rb' + - 'ee/lib/gitlab/geo/health_check.rb' + - 'lib/gitlab/background_migration/populate_untracked_uploads.rb' + - 'qa/qa/service/kubernetes_cluster.rb' + - 'spec/features/merge_request/user_sees_diff_spec.rb' + - 'spec/lib/gitlab/asciidoc_spec.rb' + - 'spec/lib/gitlab/checks/project_moved_spec.rb' + - 'spec/rubocop/cop/active_record_association_reload_spec.rb' + - 'spec/services/task_list_toggle_service_spec.rb' + +# Offense count: 14 +# Cop supports --auto-correct. +Layout/ClosingParenthesisIndentation: + Exclude: + - 'db/post_migrate/20180704145007_update_project_indexes.rb' + - 'ee/db/geo/migrate/20180405074130_add_partial_index_project_repository_verification.rb' + - 'ee/db/migrate/20180308234102_add_partial_index_to_project_repository_states_checksum_columns.rb' + - 'ee/db/post_migrate/20180605213516_fix_partial_index_to_project_repository_states_checksum_columns.rb' + - 'ee/lib/ee/gitlab/usage_data.rb' + - 'spec/services/issues/resolve_discussions_spec.rb' + - 'spec/services/projects/update_service_spec.rb' + - 'spec/support/helpers/stub_object_storage.rb' + - 'spec/workers/remove_unreferenced_lfs_objects_worker_spec.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: leading, trailing +Layout/DotPosition: + Exclude: + - 'app/models/concerns/relative_positioning.rb' + - 'app/models/group.rb' + +# Offense count: 69 +# Cop supports --auto-correct. +Layout/EmptyLinesAroundArguments: + Exclude: + - 'app/models/concerns/discussion_on_diff.rb' + - 'app/models/concerns/resolvable_discussion.rb' + - 'app/models/diff_discussion.rb' + - 'app/models/discussion.rb' + - 'ee/app/helpers/license_helper.rb' + - 'ee/spec/models/geo/project_registry_spec.rb' + - 'lib/banzai/pipeline/broadcast_message_pipeline.rb' + - 'lib/banzai/pipeline/gfm_pipeline.rb' + - 'lib/banzai/pipeline/single_line_pipeline.rb' + - 'spec/features/markdown/copy_as_gfm_spec.rb' + +# Offense count: 160 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, IndentationWidth. # SupportedStyles: special_inside_parentheses, consistent, align_brackets -Layout/IndentArray: +Layout/IndentFirstArrayElement: Enabled: false -# Offense count: 237 +# Offense count: 631 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, IndentationWidth. # SupportedStyles: special_inside_parentheses, consistent, align_braces -Layout/IndentHash: +Layout/IndentFirstHashElement: Enabled: false -# Offense count: 93 +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: consistent, align_parentheses +Layout/IndentFirstParameter: + Exclude: + - 'app/models/ci/pipeline_schedule.rb' + - 'lib/gitlab/cross_project_access.rb' + - 'lib/gitlab/data_builder/push.rb' + - 'spec/support/helpers/repo_helpers.rb' + - 'spec/support/helpers/stub_object_storage.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +Layout/LeadingBlankLines: + Exclude: + - 'app/workers/update_project_statistics_worker.rb' + - 'db/migrate/20161007073613_create_user_activities.rb' + - 'ee/spec/helpers/boards_helper_spec.rb' + - 'lib/tasks/yarn.rake' + - 'spec/javascripts/fixtures/merge_requests_diffs.rb' + +# Offense count: 30 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented +Layout/MultilineOperationIndentation: + Enabled: false + +# Offense count: 13 +# Cop supports --auto-correct. +Layout/RescueEnsureAlignment: + Exclude: + - 'app/models/blob_viewer/dependency_manager.rb' + - 'app/models/ci/pipeline.rb' + - 'app/models/project.rb' + - 'app/services/prometheus/proxy_service.rb' + - 'app/workers/delete_stored_files_worker.rb' + - 'app/workers/reactive_caching_worker.rb' + - 'config/initializers/1_settings.rb' + - 'config/initializers/trusted_proxies.rb' + - 'ee/db/migrate/20151113115819_canonicalize_kerberos_identities.rb' + - 'lib/gitlab/background_migration/archive_legacy_traces.rb' + - 'lib/gitlab/highlight.rb' + - 'lib/tasks/gitlab/lfs/migrate.rake' + +# Offense count: 344 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: require_no_space, require_space Layout/SpaceInLambdaLiteral: Enabled: false -# Offense count: 327 +# Offense count: 583 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters. # SupportedStyles: space, no_space @@ -44,20 +156,27 @@ Layout/SpaceInLambdaLiteral: Layout/SpaceInsideBlockBraces: Enabled: false -# Offense count: 156 +# Offense count: 255 # Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: space, no_space Layout/SpaceInsideParens: Enabled: false -# Offense count: 26 +# Offense count: 1 +Lint/DisjunctiveAssignmentInConstructor: + Exclude: + - 'app/models/uploads/base.rb' + +# Offense count: 19 Lint/DuplicateMethods: Exclude: - - 'app/models/application_setting.rb' - 'app/models/commit.rb' - 'app/models/note.rb' - - 'app/services/merge_requests/merge_service.rb' - 'lib/bitbucket/representation/repo.rb' - 'lib/declarative_policy/base.rb' + - 'lib/gitlab/auth/ldap/person.rb' + - 'lib/gitlab/auth/o_auth/user.rb' - 'lib/gitlab/ci/build/artifacts/metadata/entry.rb' - 'lib/gitlab/cycle_analytics/base_event_fetcher.rb' - 'lib/gitlab/diff/formatters/base_formatter.rb' @@ -65,22 +184,34 @@ Lint/DuplicateMethods: - 'lib/gitlab/git/repository.rb' - 'lib/gitlab/git/tree.rb' - 'lib/gitlab/git/wiki_page.rb' - - 'lib/gitlab/auth/ldap/person.rb' - - 'lib/gitlab/auth/o_auth/user.rb' -# Offense count: 4 +# Offense count: 2 Lint/InterpolationCheck: Exclude: - 'spec/features/issues/filtered_search/filter_issues_spec.rb' - - 'spec/features/users_spec.rb' - 'spec/services/quick_actions/interpret_service_spec.rb' -# Offense count: 206 +# Offense count: 326 # Configuration parameters: MaximumRangeSize. Lint/MissingCopEnableDirective: Enabled: false -# Offense count: 9 +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: Whitelist. +# Whitelist: present?, blank?, presence, try, try! +Lint/SafeNavigationConsistency: + Exclude: + - 'lib/gitlab/gpg/commit.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Lint/ToJSON: + Exclude: + - 'lib/gitlab/cycle_analytics/usage_data.rb' + - 'lib/gitlab/template/base_template.rb' + +# Offense count: 7 Lint/UriEscapeUnescape: Exclude: - 'app/controllers/application_controller.rb' @@ -88,13 +219,25 @@ Lint/UriEscapeUnescape: - 'spec/lib/google_api/auth_spec.rb' - 'spec/requests/api/files_spec.rb' - 'spec/requests/api/internal_spec.rb' - - 'spec/requests/api/issues_spec.rb' # Offense count: 1 -# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. +# Configuration parameters: CheckForMethodsWithNoSideEffects. +Lint/Void: + Exclude: + - 'lib/gitlab/git/diff_collection.rb' + +# Offense count: 158 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. # URISchemes: http, https Metrics/LineLength: - Max: 1310 + Max: 176 + +# Offense count: 94 +# Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms. +# AllowedAcronyms: CLI, DSL, ACL, API, ASCII, CPU, CSS, DNS, EOF, GUID, HTML, HTTP, HTTPS, ID, IP, JSON, LHS, QPS, RAM, RHS, RPC, SLA, SMTP, SQL, SSH, TCP, TLS, TTL, UDP, UI, UID, UUID, URI, URL, UTF8, VM, XML, XMPP, XSRF, XSS +Naming/FileName: + Enabled: false # Offense count: 11 # Configuration parameters: EnforcedStyle. @@ -107,61 +250,85 @@ Naming/HeredocDelimiterCase: - 'spec/support/helpers/repo_helpers.rb' - 'spec/support/helpers/seed_repo.rb' -# Offense count: 112 +# Offense count: 197 # Configuration parameters: Blacklist. -# Blacklist: END, (?-mix:EO[A-Z]{1}) +# Blacklist: (?-mix:(^|\s)(EO[A-Z]{1}|END)(\s|$)) Naming/HeredocDelimiterNaming: Enabled: false +# Offense count: 125 +# Cop supports --auto-correct. +# Configuration parameters: PreferredName. +Naming/RescuedExceptionsVariableName: + Enabled: false + +# Offense count: 6 +# Cop supports --auto-correct. +Performance/InefficientHashSearch: + Exclude: + - 'app/controllers/concerns/sessionless_authentication.rb' + - 'app/models/note.rb' + - 'app/models/user_preference.rb' + - 'ee/app/models/ee/project.rb' + - 'lib/gitlab/import_export/members_mapper.rb' + - 'qa/spec/spec_helper.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Performance/ReverseEach: + Exclude: + - 'app/models/commit.rb' + - 'db/migrate/20190222051615_add_indexes_for_merge_request_diffs_query.rb' + - 'lib/gitlab/profiler.rb' -# Offense count: 3821 +# Offense count: 7081 # Configuration parameters: Prefixes. # Prefixes: when, with, without RSpec/ContextWording: Enabled: false -# Offense count: 293 +# Offense count: 407 +# Cop supports --auto-correct. RSpec/EmptyLineAfterFinalLet: Enabled: false -# Offense count: 188 +# Offense count: 232 +# Cop supports --auto-correct. RSpec/EmptyLineAfterSubject: Enabled: false -# Offense count: 258 +# Offense count: 719 +# Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: method_call, block RSpec/ExpectChange: Enabled: false -# Offense count: 221 +# Offense count: 512 RSpec/ExpectInHook: Enabled: false -# Offense count: 19 +# Offense count: 10 +# Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: it_behaves_like, it_should_behave_like RSpec/ItBehavesLike: Exclude: - 'spec/lib/gitlab/git/commit_spec.rb' - 'spec/lib/gitlab/git/repository_spec.rb' - - 'spec/lib/gitlab/shell_spec.rb' - 'spec/services/notification_service_spec.rb' - - 'spec/workers/git_garbage_collect_worker_spec.rb' -# Offense count: 5 +# Offense count: 3 RSpec/IteratedExpectation: Exclude: - 'spec/features/admin/admin_settings_spec.rb' - - 'spec/features/merge_requests/diff_notes_resolve_spec.rb' - - 'spec/features/projects/awards/user_interacts_with_awards_in_issue_spec.rb' - 'spec/lib/gitlab/gitlab_import/client_spec.rb' - 'spec/lib/gitlab/legacy_github_import/client_spec.rb' -# Offense count: 75 +# Offense count: 68 +# Cop supports --auto-correct. RSpec/LetBeforeExamples: Exclude: - - 'spec/controllers/projects/commit_controller_spec.rb' - 'spec/lib/banzai/filter/issue_reference_filter_spec.rb' - 'spec/lib/banzai/filter/user_reference_filter_spec.rb' - 'spec/lib/gitlab/email/handler/create_issue_handler_spec.rb' @@ -170,12 +337,11 @@ RSpec/LetBeforeExamples: - 'spec/models/commit_range_spec.rb' - 'spec/models/milestone_spec.rb' - 'spec/models/project_services/packagist_service_spec.rb' - - 'spec/models/repository_spec.rb' - 'spec/rubocop/cop/migration/update_column_in_batches_spec.rb' - 'spec/serializers/pipeline_details_entity_spec.rb' - - 'spec/views/ci/lints/show.html.haml_spec.rb' # Offense count: 1 +# Cop supports --auto-correct. RSpec/MultipleSubjects: Exclude: - 'spec/services/merge_requests/create_from_issue_service_spec.rb' @@ -188,95 +354,136 @@ RSpec/OverwritingSetup: - 'spec/services/merge_requests/add_todo_when_build_fails_service_spec.rb' - 'spec/services/notes/quick_actions_service_spec.rb' -# Offense count: 965 +# Offense count: 1828 +# Cop supports --auto-correct. # Configuration parameters: Strict, EnforcedStyle. # SupportedStyles: inflected, explicit RSpec/PredicateMatcher: Enabled: false -# Offense count: 35 +# Offense count: 57 RSpec/RepeatedExample: Enabled: false -# Offense count: 140 +# Offense count: 474 +# Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: and_return, block RSpec/ReturnFromStub: Enabled: false -# Offense count: 112 +# Offense count: 188 RSpec/ScatteredLet: Enabled: false -# Offense count: 22 +# Offense count: 10 RSpec/ScatteredSetup: Exclude: - 'spec/controllers/projects/templates_controller_spec.rb' - 'spec/lib/gitlab/bitbucket_import/importer_spec.rb' - - 'spec/lib/gitlab/git/env_spec.rb' - 'spec/requests/api/jobs_spec.rb' - 'spec/services/projects/create_service_spec.rb' # Offense count: 1 +# Cop supports --auto-correct. RSpec/SharedContext: Exclude: - 'spec/features/admin/admin_groups_spec.rb' -# Offense count: 5 +# Offense count: 4 RSpec/VoidExpect: Exclude: - - 'spec/features/projects/artifacts/download_spec.rb' - 'spec/features/projects/services/user_activates_mattermost_slash_command_spec.rb' - 'spec/models/ci/group_spec.rb' - 'spec/models/ci/runner_spec.rb' - 'spec/services/users/destroy_service_spec.rb' -# Offense count: 41 +# Offense count: 8 +# Cop supports --auto-correct. +Rails/BelongsTo: + Exclude: + - 'app/models/deployment.rb' + - 'app/models/environment.rb' + - 'ee/app/models/prometheus_alert.rb' + - 'ee/app/models/prometheus_alert_event.rb' + +# Offense count: 80 # Configuration parameters: Include. # Include: db/migrate/*.rb Rails/CreateTableWithTimestamps: Enabled: false -# Offense count: 155 +# Offense count: 222 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: slashes, arguments Rails/FilePath: Enabled: false -# Offense count: 121 +# Offense count: 167 # Configuration parameters: Include. # Include: app/models/**/*.rb Rails/HasManyOrHasOneDependent: Enabled: false -# Offense count: 157 +# Offense count: 40 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: numeric, symbolic +Rails/HttpStatus: + Enabled: false + +# Offense count: 2 +# Configuration parameters: Include. +# Include: app/controllers/**/*.rb +Rails/IgnoredSkipActionFilterOption: + Exclude: + - 'app/controllers/projects/snippets_controller.rb' + - 'app/controllers/snippets_controller.rb' + +# Offense count: 87 # Configuration parameters: Include. # Include: app/models/**/*.rb Rails/InverseOf: Enabled: false -# Offense count: 48 +# Offense count: 46 # Configuration parameters: Include. # Include: app/controllers/**/*.rb Rails/LexicallyScopedActionFilter: Enabled: false -# Offense count: 14 +# Offense count: 4 +# Cop supports --auto-correct. +Rails/LinkToBlank: + Exclude: + - 'app/helpers/projects_helper.rb' + - 'app/helpers/wiki_helper.rb' + - 'ee/app/helpers/ee/user_callouts_helper.rb' + - 'ee/app/helpers/license_helper.rb' + +# Offense count: 11 # Cop supports --auto-correct. Rails/Presence: Exclude: - - 'app/controllers/projects/blob_controller.rb' - 'app/models/ci/pipeline.rb' - 'app/models/clusters/platforms/kubernetes.rb' - 'app/models/concerns/mentionable.rb' - - 'app/models/concerns/token_authenticatable.rb' - 'app/models/project_services/hipchat_service.rb' - 'app/models/project_services/irker_service.rb' - 'app/models/project_services/jira_service.rb' - 'app/models/project_services/kubernetes_service.rb' - 'app/models/project_services/packagist_service.rb' - 'app/models/wiki_page.rb' - - 'lib/gitlab/git/hook.rb' - 'lib/gitlab/github_import/importer/releases_importer.rb' +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: Include. +# Include: app/models/**/*.rb +Rails/RedundantAllowNil: + Exclude: + - 'app/models/application_setting.rb' + # Offense count: 2 # Configuration parameters: Include. # Include: db/migrate/*.rb @@ -284,8 +491,8 @@ Rails/ReversibleMigration: Exclude: - 'db/migrate/20160824103857_drop_unused_ci_tables.rb' -# Offense count: 446 -# Configuration parameters: Blacklist. +# Offense count: 860 +# Configuration parameters: Blacklist, Whitelist. # Blacklist: decrement!, decrement_counter, increment!, increment_counter, toggle!, touch, update_all, update_attribute, update_column, update_columns, update_counters Rails/SkipsModelValidations: Enabled: false @@ -297,21 +504,25 @@ Rails/UnknownEnv: Exclude: - 'db/migrate/20171124125748_populate_missing_merge_request_statuses.rb' -# Offense count: 13 +# Offense count: 11 # Cop supports --auto-correct. Security/YAMLLoad: Exclude: - - 'config/initializers/carrierwave.rb' - 'lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb' - 'lib/gitlab/redis/wrapper.rb' - 'lib/system_check/incoming_email/imap_authentication_check.rb' - 'spec/config/mail_room_spec.rb' - 'spec/initializers/secret_token_spec.rb' - 'spec/lib/gitlab/prometheus/additional_metrics_parser_spec.rb' - - 'spec/models/clusters/platforms/kubernetes_spec.rb' - 'spec/models/project_services/kubernetes_service_spec.rb' -# Offense count: 64 +# Offense count: 34 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: inline, group +Style/AccessModifierDeclarations: + Enabled: false + +# Offense count: 121 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: percent_q, bare_percent @@ -324,17 +535,13 @@ Style/CommentedKeyword: - 'lib/tasks/gitlab/backup.rake' - 'spec/tasks/gitlab/backup_rake_spec.rb' -# Offense count: 30 -Style/DateTime: - Enabled: false - # Offense count: 1 # Cop supports --auto-correct. Style/Dir: Exclude: - 'qa/qa.rb' -# Offense count: 9 +# Offense count: 7 # Cop supports --auto-correct. Style/EachWithObject: Exclude: @@ -345,46 +552,42 @@ Style/EachWithObject: - 'lib/gitlab/i18n/po_linter.rb' - 'lib/gitlab/import_export/members_mapper.rb' - 'lib/gitlab/import_export/relation_factory.rb' - - 'scripts/static-analysis' -# Offense count: 24 +# Offense count: 34 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: empty, nil, both Style/EmptyElse: Enabled: false -# Offense count: 14 +# Offense count: 11 # Cop supports --auto-correct. Style/EmptyLambdaParameter: Exclude: - 'app/models/ci/build.rb' - 'app/models/ci/runner.rb' -# Offense count: 12 +# Offense count: 9 # Cop supports --auto-correct. Style/EmptyLiteral: Exclude: - - 'features/steps/project/commits/commits.rb' - 'lib/gitlab/fogbugz_import/importer.rb' - 'lib/gitlab/git/diff_collection.rb' - 'lib/gitlab/gitaly_client.rb' - - 'scripts/trigger-build' - - 'spec/features/merge_requests/versions_spec.rb' - 'spec/helpers/merge_requests_helper_spec.rb' - 'spec/lib/gitlab/request_context_spec.rb' - 'spec/lib/gitlab/workhorse_spec.rb' - 'spec/requests/api/jobs_spec.rb' - 'spec/support/shared_examples/chat_slash_commands_shared_examples.rb' -# Offense count: 102 +# Offense count: 180 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: compact, expanded Style/EmptyMethod: Enabled: false -# Offense count: 23 +# Offense count: 40 # Cop supports --auto-correct. Style/Encoding: Enabled: false @@ -394,52 +597,52 @@ Style/EvalWithLocation: Exclude: - 'app/models/service.rb' -# Offense count: 35 +# Offense count: 203 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: format, sprintf, percent Style/FormatString: Enabled: false -# Offense count: 384 +# Offense count: 669 # Configuration parameters: MinBodyLength. Style/GuardClause: Enabled: false -# Offense count: 22 +# Offense count: 27 Style/IfInsideElse: Enabled: false -# Offense count: 809 +# Offense count: 1346 # Cop supports --auto-correct. Style/IfUnlessModifier: Enabled: false -# Offense count: 75 +# Offense count: 186 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: line_count_dependent, lambda, literal Style/Lambda: Enabled: false -# Offense count: 11 +# Offense count: 3 # Cop supports --auto-correct. Style/LineEndConcatenation: Exclude: - - 'app/helpers/tree_helper.rb' - - 'spec/features/issuables/markdown_references_spec.rb' - - 'spec/lib/gitlab/checks/project_moved_spec.rb' - 'spec/lib/gitlab/gfm/reference_rewriter_spec.rb' - 'spec/lib/gitlab/incoming_email_spec.rb' # Offense count: 18 -Style/MethodMissing: +Style/MethodMissingSuper: Enabled: false -# Offense count: 7 +# Offense count: 18 +Style/MissingRespondToMissing: + Enabled: false + +# Offense count: 6 Style/MixinUsage: Exclude: - - 'features/support/env.rb' - 'spec/factories/ci/builds.rb' - 'spec/factories/ci/job_artifacts.rb' - 'spec/factories/lfs_objects.rb' @@ -447,59 +650,63 @@ Style/MixinUsage: - 'spec/lib/gitlab/import_export/project_tree_restorer_spec.rb' - 'spec/lib/gitlab/import_export/version_checker_spec.rb' -# Offense count: 6 +# Offense count: 4 # Cop supports --auto-correct. Style/MultilineIfModifier: Exclude: - 'app/helpers/snippets_helper.rb' - 'app/models/project_wiki.rb' - 'app/services/ci/process_pipeline_service.rb' - - 'app/services/create_deployment_service.rb' - 'lib/api/commit_statuses.rb' - - 'lib/gitlab/ci/trace.rb' -# Offense count: 25 +# Offense count: 72 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: literals, strict +Style/MutableConstant: + Enabled: false + +# Offense count: 28 # Cop supports --auto-correct. # Configuration parameters: Whitelist. # Whitelist: be, be_a, be_an, be_between, be_falsey, be_kind_of, be_instance_of, be_truthy, be_within, eq, eql, end_with, include, match, raise_error, respond_to, start_with Style/NestedParenthesizedCalls: Enabled: false -# Offense count: 19 +# Offense count: 31 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, MinBodyLength. # SupportedStyles: skip_modifier_ifs, always Style/Next: Enabled: false -# Offense count: 61 +# Offense count: 67 # Cop supports --auto-correct. # Configuration parameters: EnforcedOctalStyle. # SupportedOctalStyles: zero_with_o, zero_only Style/NumericLiteralPrefix: Enabled: false -# Offense count: 114 +# Offense count: 186 # Cop supports --auto-correct. -# Configuration parameters: AutoCorrect, EnforcedStyle. +# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods. # SupportedStyles: predicate, comparison Style/NumericPredicate: Enabled: false -# Offense count: 4 +# Offense count: 2 # Cop supports --auto-correct. Style/OrAssignment: Exclude: - - 'app/models/concerns/token_authenticatable.rb' - 'lib/api/commit_statuses.rb' - 'lib/gitlab/project_transfer.rb' -# Offense count: 50 +# Offense count: 79 # Cop supports --auto-correct. Style/ParallelAssignment: Enabled: false -# Offense count: 917 +# Offense count: 1390 # Cop supports --auto-correct. # Configuration parameters: PreferredDelimiters. Style/PercentLiteralDelimiters: @@ -521,25 +728,20 @@ Style/PerlBackrefs: - 'lib/gitlab/search_results.rb' - 'lib/gitlab/sherlock/query.rb' -# Offense count: 87 +# Offense count: 129 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: compact, exploded Style/RaiseArgs: Enabled: false -# Offense count: 9 +# Offense count: 3 # Cop supports --auto-correct. Style/RedundantBegin: Exclude: - - 'app/controllers/projects/clusters/gcp_controller.rb' - 'app/models/merge_request.rb' - 'app/services/projects/import_service.rb' - - 'lib/api/branches.rb' - - 'lib/gitlab/current_settings.rb' - - 'lib/gitlab/git/commit.rb' - 'lib/gitlab/health_checks/base_abstract_check.rb' - - 'lib/tasks/gitlab/task_helpers.rb' # Offense count: 1 # Cop supports --auto-correct. @@ -547,7 +749,7 @@ Style/RedundantConditional: Exclude: - 'lib/system_check/helpers.rb' -# Offense count: 57 +# Offense count: 360 # Cop supports --auto-correct. Style/RedundantFreeze: Enabled: false @@ -567,41 +769,36 @@ Style/RedundantReturn: - 'lib/gitlab/utils.rb' - 'lib/google_api/auth.rb' -# Offense count: 460 +# Offense count: 700 # Cop supports --auto-correct. Style/RedundantSelf: Enabled: false -# Offense count: 142 +# Offense count: 28 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, AllowInnerSlashes. # SupportedStyles: slashes, percent_r, mixed Style/RegexpLiteral: - Enabled: true - EnforcedStyle: mixed - AllowInnerSlashes: false + Enabled: false -# Offense count: 36 +# Offense count: 41 # Cop supports --auto-correct. Style/RescueModifier: Enabled: false -# Offense count: 107 +# Offense count: 197 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: implicit, explicit Style/RescueStandardError: Enabled: false -# Offense count: 8 +# Offense count: 5 # Cop supports --auto-correct. Style/SelfAssignment: Exclude: - 'app/models/concerns/bulk_member_access_load.rb' - 'app/serializers/base_serializer.rb' - - 'app/services/notification_service.rb' - - 'lib/api/runners.rb' - - 'spec/features/merge_requests/diff_notes_resolve_spec.rb' - 'spec/features/projects/clusters/interchangeability_spec.rb' - 'spec/support/import_export/configuration_helper.rb' @@ -612,7 +809,7 @@ Style/SingleLineMethods: Exclude: - 'lib/gitlab/ci/ansi2html.rb' -# Offense count: 66 +# Offense count: 91 # Cop supports --auto-correct. # Configuration parameters: . # SupportedStyles: use_perl_names, use_english_names @@ -625,21 +822,21 @@ Style/StderrPuts: Exclude: - 'config/initializers/rspec_profiling.rb' -# Offense count: 45 +# Offense count: 65 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle. # SupportedStyles: single_quotes, double_quotes Style/StringLiteralsInInterpolation: Enabled: false -# Offense count: 106 +# Offense count: 187 # Cop supports --auto-correct. # Configuration parameters: IgnoredMethods. # IgnoredMethods: respond_to, define_method Style/SymbolProc: Enabled: false -# Offense count: 9 +# Offense count: 7 # Cop supports --auto-correct. # Configuration parameters: EnforcedStyle, AllowSafeAssignment. # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex @@ -647,45 +844,65 @@ Style/TernaryParentheses: Exclude: - 'app/finders/projects_finder.rb' - 'app/helpers/namespaces_helper.rb' - - 'features/support/capybara.rb' - 'lib/gitlab/ci/build/artifacts/metadata/entry.rb' - 'spec/requests/api/pipeline_schedules_spec.rb' - 'spec/support/capybara.rb' -# Offense count: 17 +# Offense count: 3 # Cop supports --auto-correct. -# Configuration parameters: AllowNamedUnderscoreVariables. -Style/TrailingUnderscoreVariable: +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInArguments: Exclude: - - 'app/controllers/admin/background_jobs_controller.rb' - - 'app/controllers/invites_controller.rb' - - 'app/helpers/tab_helper.rb' - - 'lib/backup/manager.rb' - - 'lib/gitlab/logger.rb' - - 'lib/gitlab/upgrader.rb' - - 'lib/system_check/app/migrations_are_up_check.rb' - - 'lib/system_check/incoming_email/mail_room_running_check.rb' - - 'lib/tasks/gitlab/check.rake' - - 'lib/tasks/gitlab/task_helpers.rb' - - 'spec/lib/gitlab/etag_caching/middleware_spec.rb' - - 'spec/services/quick_actions/interpret_service_spec.rb' + - 'spec/features/markdown/copy_as_gfm_spec.rb' -# Offense count: 4 +# Offense count: 10 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInArrayLiteral: + Exclude: + - 'ee/spec/models/project_spec.rb' + - 'spec/lib/gitlab/diff/position_tracer_spec.rb' + - 'spec/lib/gitlab/metrics/dashboard/processor_spec.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInHashLiteral: + Exclude: + - 'lib/gitlab/ci/ansi2html.rb' + - 'lib/gitlab/kubernetes.rb' + +# Offense count: 2 # Cop supports --auto-correct. Style/UnlessElse: Exclude: - 'lib/backup/manager.rb' - 'lib/gitlab/project_search_results.rb' - - 'lib/tasks/gitlab/check.rake' - - 'spec/features/issues/award_emoji_spec.rb' -# Offense count: 31 +# Offense count: 10 +# Cop supports --auto-correct. +Style/UnneededCondition: + Exclude: + - 'app/helpers/button_helper.rb' + - 'app/helpers/environment_helper.rb' + - 'app/models/project.rb' + - 'app/services/issuable/clone/base_service.rb' + - 'app/services/prometheus/adapter_service.rb' + - 'lib/gitlab/email/message/repository_push.rb' + - 'lib/gitlab/prometheus_client.rb' + - 'spec/lib/rspec_flaky/flaky_example_spec.rb' + +# Offense count: 73 # Cop supports --auto-correct. Style/UnneededInterpolation: Enabled: false -# Offense count: 22840 -# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. -# URISchemes: http, https -Metrics/LineLength: - Max: 1310 +# Offense count: 2 +# Cop supports --auto-correct. +Style/UnneededSort: + Exclude: + - 'app/models/concerns/resolvable_discussion.rb' + - 'lib/gitlab/highlight.rb' @@ -349,13 +349,14 @@ group :development, :test do gem 'spring', '~> 2.0.0' gem 'spring-commands-rspec', '~> 1.0.4' - gem 'gitlab-styles', '~> 2.5', require: false + gem 'gitlab-styles', '~> 2.6', require: false # Pin these dependencies, otherwise a new rule could break the CI pipelines - gem 'rubocop', '~> 0.54.0' + gem 'rubocop', '~> 0.68.1' + gem 'rubocop-performance', '~> 1.1.0' gem 'rubocop-rspec', '~> 1.22.1' gem 'scss_lint', '~> 0.56.0', require: false - gem 'haml_lint', '~> 0.28.0', require: false + gem 'haml_lint', '~> 0.30.0', require: false gem 'simplecov', '~> 0.14.0', require: false gem 'bundler-audit', '~> 0.5.0', require: false diff --git a/Gemfile.lock b/Gemfile.lock index e02de8292bd..bdcefd2e2b5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -297,9 +297,10 @@ GEM gitlab-markup (1.7.0) gitlab-sidekiq-fetcher (0.4.0) sidekiq (~> 5) - gitlab-styles (2.5.2) - rubocop (~> 0.54.0) + gitlab-styles (2.6.2) + rubocop (~> 0.68.1) rubocop-gitlab-security (~> 0.1.0) + rubocop-performance (~> 1.1.0) rubocop-rspec (~> 1.19) gitlab_omniauth-ldap (2.1.1) net-ldap (~> 0.16) @@ -357,7 +358,7 @@ GEM haml (5.0.4) temple (>= 0.8.0) tilt - haml_lint (0.28.0) + haml_lint (0.30.0) haml (>= 4.0, < 5.1) rainbow rake (>= 10, < 13) @@ -407,6 +408,7 @@ GEM jaeger-client (0.10.0) opentracing (~> 0.3) thrift + jaro_winkler (1.5.2) jira-ruby (1.4.1) activesupport multipart-post @@ -585,7 +587,7 @@ GEM rubypants (~> 0.2) orm_adapter (0.5.0) os (1.0.0) - parallel (1.12.1) + parallel (1.17.0) parser (2.5.3.0) ast (~> 2.4.0) parslet (1.8.2) @@ -615,7 +617,6 @@ GEM pg (1.1.4) po_to_json (1.0.1) json (>= 1.6.0) - powerpack (0.1.1) premailer (1.10.4) addressable css_parser (>= 1.4.10) @@ -787,15 +788,17 @@ GEM pg rails sqlite3 - rubocop (0.54.0) + rubocop (0.68.1) + jaro_winkler (~> 1.5.1) parallel (~> 1.10) - parser (>= 2.5) - powerpack (~> 0.1) + parser (>= 2.5, != 2.5.1.1) rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.0, >= 1.0.1) + unicode-display_width (>= 1.4.0, < 1.6) rubocop-gitlab-security (0.1.1) rubocop (>= 0.51) + rubocop-performance (1.1.0) + rubocop (>= 0.67.0) rubocop-rspec (1.22.2) rubocop (>= 0.52.1) ruby-enum (0.7.2) @@ -803,7 +806,7 @@ GEM ruby-fogbugz (0.2.1) crack (~> 0.4) ruby-prof (0.17.0) - ruby-progressbar (1.9.0) + ruby-progressbar (1.10.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) ruby_parser (3.11.0) @@ -929,7 +932,7 @@ GEM unf (0.1.4) unf_ext unf_ext (0.0.7.5) - unicode-display_width (1.3.2) + unicode-display_width (1.5.0) unicorn (5.4.1) kgio (~> 2.6) raindrops (~> 0.7) @@ -1065,7 +1068,7 @@ DEPENDENCIES gitlab-labkit (~> 0.2.0) gitlab-markup (~> 1.7.0) gitlab-sidekiq-fetcher (~> 0.4.0) - gitlab-styles (~> 2.5) + gitlab-styles (~> 2.6) gitlab_omniauth-ldap (~> 2.1.1) gon (~> 6.2) google-api-client (~> 0.23) @@ -1078,7 +1081,7 @@ DEPENDENCIES graphiql-rails (~> 1.4.10) graphql (~> 1.8.0) grpc (~> 1.19.0) - haml_lint (~> 0.28.0) + haml_lint (~> 0.30.0) hamlit (~> 2.8.8) hangouts-chat (~> 0.0.5) hashie-forbidden_attributes @@ -1171,7 +1174,8 @@ DEPENDENCIES rspec-set (~> 0.1.3) rspec_junit_formatter rspec_profiling (~> 0.0.5) - rubocop (~> 0.54.0) + rubocop (~> 0.68.1) + rubocop-performance (~> 1.1.0) rubocop-rspec (~> 1.22.1) ruby-fogbugz (~> 0.2.1) ruby-prof (~> 0.17.0) diff --git a/app/controllers/help_controller.rb b/app/controllers/help_controller.rb index 10cdce98437..837c26c630a 100644 --- a/app/controllers/help_controller.rb +++ b/app/controllers/help_controller.rb @@ -7,7 +7,7 @@ class HelpController < ApplicationController # Taken from Jekyll # https://github.com/jekyll/jekyll/blob/3.5-stable/lib/jekyll/document.rb#L13 - YAML_FRONT_MATTER_REGEXP = /\A(---\s*\n.*?\n?)^((---|\.\.\.)\s*$\n?)/m + YAML_FRONT_MATTER_REGEXP = /\A(---\s*\n.*?\n?)^((---|\.\.\.)\s*$\n?)/m.freeze def index # Remove YAML frontmatter so that it doesn't look weird diff --git a/app/controllers/import/bitbucket_server_controller.rb b/app/controllers/import/bitbucket_server_controller.rb index 643a3bfed1f..f71ea8642cd 100644 --- a/app/controllers/import/bitbucket_server_controller.rb +++ b/app/controllers/import/bitbucket_server_controller.rb @@ -15,8 +15,8 @@ class Import::BitbucketServerController < Import::BaseController # (https://community.atlassian.com/t5/Answers-Developer-Questions/stash-repository-names/qaq-p/499054) # # Bitbucket Server starts personal project names with a tilde. - VALID_BITBUCKET_PROJECT_CHARS = /\A~?[\w\-\.\s]+\z/ - VALID_BITBUCKET_CHARS = /\A[\w\-\.\s]+\z/ + VALID_BITBUCKET_PROJECT_CHARS = /\A~?[\w\-\.\s]+\z/.freeze + VALID_BITBUCKET_CHARS = /\A[\w\-\.\s]+\z/.freeze def new end diff --git a/app/helpers/auth_helper.rb b/app/helpers/auth_helper.rb index b4ee648361c..076976175a9 100644 --- a/app/helpers/auth_helper.rb +++ b/app/helpers/auth_helper.rb @@ -2,7 +2,7 @@ module AuthHelper PROVIDERS_WITH_ICONS = %w(twitter github gitlab bitbucket google_oauth2 facebook azure_oauth2 authentiq).freeze - LDAP_PROVIDER = /\Aldap/ + LDAP_PROVIDER = /\Aldap/.freeze def ldap_enabled? Gitlab::Auth::LDAP::Config.enabled? diff --git a/app/helpers/sidekiq_helper.rb b/app/helpers/sidekiq_helper.rb index 32bf3526571..6326d98461e 100644 --- a/app/helpers/sidekiq_helper.rb +++ b/app/helpers/sidekiq_helper.rb @@ -8,7 +8,7 @@ module SidekiqHelper (?<state>[DIEKNRSTVWXZNLpsl\+<>/\d]+)\s+ (?<start>.+?)\s+ (?<command>(?:ruby\d+:\s+)?sidekiq.*\].*) - \z}x + \z}x.freeze def parse_sidekiq_ps(line) match = line.strip.match(SIDEKIQ_PS_REGEXP) diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index ee12a1d09f3..e51619b0f9c 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -8,7 +8,7 @@ module ApplicationSettingImplementation \s # any whitespace character | # or [\r\n] # any number of newline characters - }x + }x.freeze # Setting a key restriction to `-1` means that all keys of this type are # forbidden. diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 920b1d092dd..08ca86bc902 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -28,12 +28,12 @@ class CommitRange # The beginning and ending refs can be named or SHAs, and # the range notation can be double- or triple-dot. - REF_PATTERN = /[0-9a-zA-Z][0-9a-zA-Z_.-]*[0-9a-zA-Z\^]/ - PATTERN = /#{REF_PATTERN}\.{2,3}#{REF_PATTERN}/ + REF_PATTERN = /[0-9a-zA-Z][0-9a-zA-Z_.-]*[0-9a-zA-Z\^]/.freeze + PATTERN = /#{REF_PATTERN}\.{2,3}#{REF_PATTERN}/.freeze # In text references, the beginning and ending refs can only be SHAs # between 7 and 40 hex characters. - STRICT_PATTERN = /\h{7,40}\.{2,3}\h{7,40}/ + STRICT_PATTERN = /\h{7,40}\.{2,3}\h{7,40}/.freeze def self.reference_prefix '@' diff --git a/app/models/concerns/maskable.rb b/app/models/concerns/maskable.rb index 8793f0ec965..2943872ffab 100644 --- a/app/models/concerns/maskable.rb +++ b/app/models/concerns/maskable.rb @@ -9,7 +9,7 @@ module Maskable # * No spaces # * Minimal length of 8 characters # * Absolutely no fun is allowed - REGEX = /\A\w{8,}\z/ + REGEX = /\A\w{8,}\z/.freeze included do validates :masked, inclusion: { in: [true, false] } diff --git a/app/models/concerns/redactable.rb b/app/models/concerns/redactable.rb index 5ad96d6cc46..53ae300ee2d 100644 --- a/app/models/concerns/redactable.rb +++ b/app/models/concerns/redactable.rb @@ -10,7 +10,7 @@ module Redactable extend ActiveSupport::Concern - UNSUBSCRIBE_PATTERN = %r{/sent_notifications/\h{32}/unsubscribe} + UNSUBSCRIBE_PATTERN = %r{/sent_notifications/\h{32}/unsubscribe}.freeze class_methods do def redact_field(field) diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index f147ce8ad6b..2f0e078c807 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -19,7 +19,7 @@ module Taskable \s+ # whitespace prefix has to be always presented for a list item (\[\s\]|\[[xX]\]) # checkbox (\s.+) # followed by whitespace and some text. - }x + }x.freeze def self.get_tasks(content) content.to_s.scan(ITEM_PATTERN).map do |checkbox, label| diff --git a/app/models/environment.rb b/app/models/environment.rb index 69224635e34..0eda7a2513f 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -3,8 +3,8 @@ class Environment < ApplicationRecord include Gitlab::Utils::StrongMemoize # Used to generate random suffixes for the slug - LETTERS = 'a'..'z' - NUMBERS = '0'..'9' + LETTERS = ('a'..'z').freeze + NUMBERS = ('0'..'9').freeze SUFFIX_CHARS = LETTERS.to_a + NUMBERS.to_a belongs_to :project, required: true diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 72270ee8b4f..0b4fef5eac1 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -16,7 +16,7 @@ module ErrorTracking (?<project>[^/]+)/* )? \z - }x + }x.freeze self.reactive_cache_key = ->(setting) { [setting.class.model_name.singular, setting.project_id] } diff --git a/app/services/projects/import_error_filter.rb b/app/services/projects/import_error_filter.rb index a0fc5149bb4..737b794484d 100644 --- a/app/services/projects/import_error_filter.rb +++ b/app/services/projects/import_error_filter.rb @@ -4,7 +4,7 @@ module Projects # Used by project imports, it removes any potential paths # included in an error message that could be stored in the DB class ImportErrorFilter - ERROR_MESSAGE_FILTER = /[^\s]*#{File::SEPARATOR}[^\s]*(?=(\s|\z))/ + ERROR_MESSAGE_FILTER = /[^\s]*#{File::SEPARATOR}[^\s]*(?=(\s|\z))/.freeze FILTER_MESSAGE = '[FILTERED]' def self.filter_message(message) diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index e90599f2505..6dfe2bed0ba 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -14,8 +14,8 @@ class FileUploader < GitlabUploader include ObjectStorage::Concern prepend ObjectStorage::Extension::RecordsUploads - MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} - DYNAMIC_PATH_PATTERN = %r{.*(?<secret>\h{32})/(?<identifier>.*)} + MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)}.freeze + DYNAMIC_PATH_PATTERN = %r{.*(?<secret>\h{32})/(?<identifier>.*)}.freeze after :remove, :prune_store_dir diff --git a/config/initializers/gettext_rails_i18n_patch.rb b/config/initializers/gettext_rails_i18n_patch.rb index c1342f48ebd..714dd505824 100644 --- a/config/initializers/gettext_rails_i18n_patch.rb +++ b/config/initializers/gettext_rails_i18n_patch.rb @@ -2,7 +2,7 @@ require 'gettext_i18n_rails/haml_parser' require 'gettext_i18n_rails_js/parser/javascript' require 'json' -VUE_TRANSLATE_REGEX = /((%[\w.-]+)(?:\s))?{{ (N|n|s)?__\((.*)\) }}/ +VUE_TRANSLATE_REGEX = /((%[\w.-]+)(?:\s))?{{ (N|n|s)?__\((.*)\) }}/.freeze module GettextI18nRails class HamlParser diff --git a/danger/commit_messages/Dangerfile b/danger/commit_messages/Dangerfile index 9be1ce2ff86..048c539bcf9 100644 --- a/danger/commit_messages/Dangerfile +++ b/danger/commit_messages/Dangerfile @@ -21,7 +21,7 @@ class EmojiChecker # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this # regex to save us from having to check for all possible emoji names when we # know one definitely is not included. - LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/ + LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze def initialize names = JSON.parse(File.read(DIGESTS)).keys + diff --git a/db/migrate/20140414131055_change_state_to_allow_empty_merge_request_diffs.rb b/db/migrate/20140414131055_change_state_to_allow_empty_merge_request_diffs.rb index 148b46f8830..1f9ae3f0080 100644 --- a/db/migrate/20140414131055_change_state_to_allow_empty_merge_request_diffs.rb +++ b/db/migrate/20140414131055_change_state_to_allow_empty_merge_request_diffs.rb @@ -1,4 +1,3 @@ -# rubocop:disable all class ChangeStateToAllowEmptyMergeRequestDiffs < ActiveRecord::Migration[4.2] def up change_column :merge_request_diffs, :state, :string, null: true, diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index 48f599fa7e6..4d55f2357c1 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -384,14 +384,12 @@ CAUTION: **Warning:** GitLab will not backup Docker images that are not stored o filesystem. Remember to enable backups with your object storage provider if desired. ---- +NOTE: **Note:** +`regionendpoint` is only required when configuring an S3 compatible service such as Minio. It takes a URL such as `http://127.0.0.1:9000`. **Omnibus GitLab installations** -> **Note:** -`regionendpoint` is only required when configuring an S3 compatible service such as Minio, by entering a URL such as http://127.0.0.1:9000 - -To configure the storage driver in Omnibus: +To configure the `s3` storage driver in Omnibus: 1. Edit `/etc/gitlab/gitlab.rb`: @@ -409,16 +407,14 @@ To configure the storage driver in Omnibus: 1. Save the file and [reconfigure GitLab][] for the changes to take effect. ---- - **Installations from source** Configuring the storage driver is done in your registry config YML file created when you [deployed your docker registry][registry-deploy]. -Example: +`s3` storage driver example: -``` +```yml storage: s3: accesskey: 'AKIAKIAKI' @@ -663,6 +659,37 @@ Start with a value between `25000000` (25MB) and `50000000` (50MB). 1. Save the file and [restart GitLab][] for the changes to take effect. +### Supporting older Docker clients + +As of GitLab 11.9, we began shipping version 2.7.1 of the Docker container registry, which disables the schema1 manifest by default. If you are still using older Docker clients (1.9 or older), you may experience an error pushing images. See [omnibus-4145](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4145) for more details. + +You can add a configuration option for backwards compatibility. + +**For Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + registry['compatibility_schema1_enabled'] = true + ``` + +1. Save the file and [reconfigure GitLab][] for the changes to take effect. + +--- + +**For installations from source** + +1. Edit the YML configuration file you created when you [deployed the registry][registry-deploy]. Add the following snippet: + + ```yaml + compatibility: + schema1: + enabled: true + ``` + +1. Restart the registry for the changes to take affect. + + [ce-18239]: https://gitlab.com/gitlab-org/gitlab-ce/issues/18239 [docker-insecure-self-signed]: https://docs.docker.com/registry/insecure/#use-self-signed-certificates [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/geo/replication/high_availability.md b/doc/administration/geo/replication/high_availability.md index c87c50ce1be..715a83a9ff3 100644 --- a/doc/administration/geo/replication/high_availability.md +++ b/doc/administration/geo/replication/high_availability.md @@ -101,10 +101,44 @@ documentation: ### Step 2: Configure the main read-only replica PostgreSQL database on the **secondary** node NOTE: **Note:** The following documentation assumes the database will be run on -only a single machine, rather than as a PostgreSQL cluster. +a single node only, rather than as a PostgreSQL cluster. Configure the [**secondary** database](database.md) as a read-only replica of -the **primary** database. +the **primary** database. Use the following as a guide. + +1. Edit `/etc/gitlab/gitlab.rb` in the replica database machine, and add the + following: + + ```ruby + ## + ## Configure the PostgreSQL role + ## + roles ['postgres_role'] + + ## + ## Secondary address + ## - replace '<secondary_node_ip>' with the public or VPC address of your Geo secondary node + ## - replace '<tracking_database_ip>' with the public or VPC address of your Geo tracking database node + ## + postgresql['listen_address'] = '<secondary_node_ip>' + postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32', '<tracking_database_ip>/32'] + + ## + ## Database credentials password (defined previously in primary node) + ## - replicate same values here as defined in primary node + ## + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + gitlab_rails['db_password'] = '<your_password_here>' + + ## + ## When running the Geo tracking database on a separate machine, disable it + ## here and allow connections from the tracking database host. And ensure + ## the tracking database IP is in postgresql['md5_auth_cidr_addresses'] above. + ## + geo_postgresql['enable'] = false + ``` + +After making these changes, [reconfigure GitLab][gitlab-reconfigure] so the changes take effect. If using an external PostgreSQL instance, refer also to [Geo with external PostgreSQL instances](external_database.md). @@ -124,10 +158,42 @@ Configure the tracking database. ## Enable the Geo secondary tracking database ## geo_postgresql['enable'] = true - geo_postgresql['ha'] = true + geo_postgresql['listen_address'] = '<ip_address_of_this_host>' + geo_postgresql['sql_user_password'] = '<tracking_database_password_md5_hash>' + + ## + ## Configure FDW connection to the replica database + ## + geo_secondary['db_fdw'] = true + geo_postgresql['fdw_external_password'] = '<replica_database_password_plaintext>' + geo_postgresql['md5_auth_cidr_addresses'] = ['<replica_database_ip>/32'] + gitlab_rails['db_host'] = '<replica_database_ip>' + + # Prevent reconfigure from attempting to run migrations on the replica DB + gitlab_rails['auto_migrate'] = false + + ## + ## Disable all other services that aren't needed, since we don't have a role + ## that does this. + ## + alertmanager['enable'] = false + consul['enable'] = false + gitaly['enable'] = false + gitlab_monitor['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = false + node_exporter['enable'] = false + pgbouncer_exporter['enable'] = false + postgresql['enable'] = false + prometheus['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + repmgr['enable'] = false + sidekiq['enable'] = false + unicorn['enable'] = false ``` -After making these changes [Reconfigure GitLab][gitlab-reconfigure] so the changes take effect. +After making these changes, [reconfigure GitLab][gitlab-reconfigure] so the changes take effect. If using an external PostgreSQL instance, refer also to [Geo with external PostgreSQL instances](external_database.md). @@ -189,6 +255,7 @@ following modifications: registry['uid'] = 9002 registry['gid'] = 9002 ``` + NOTE: **Note:** If you had set up PostgreSQL cluster using the omnibus package and you had set up `postgresql['sql_user_password'] = 'md5 digest of secret'` setting, keep in diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index c69ac52a41b..de61c01991b 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -1,4 +1,4 @@ -# High Availability +# Scaling and High Availability GitLab supports several different types of clustering and high-availability. The solution you choose will be based on the level of scalability and @@ -13,47 +13,173 @@ of Git, developers can still commit code locally even when GitLab is not available. However, some GitLab features such as the issue tracker and Continuous Integration are not available when GitLab is down. -**Keep in mind that all Highly Available solutions come with a trade-off between +**Keep in mind that all highly-available solutions come with a trade-off between cost/complexity and uptime**. The more uptime you want, the more complex the solution. And the more complex the solution, the more work is involved in setting up and maintaining it. High availability is not free and every HA solution should balance the costs against the benefits. -## Architecture +There are many options when choosing a highly-available GitLab architecture. We +recommend engaging with GitLab Support to choose the best architecture for your +use-case. This page contains some various options and guidelines based on +experience with GitLab.com and Enterprise Edition on-premises customers. -There are two kinds of setups: +For a detailed insight into how GitLab scales and configures GitLab.com, you can +watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) +with [John Northrup](https://gitlab.com/northrup), one of our infrastructure +engineers, and live questions coming in from some of our customers. -- active/active -- active/passive +## GitLab Components -### Active/Active +The following components need to be considered for a scaled or highly-available +environment. In many cases components can be combined on the same nodes to reduce +complexity. -This architecture scales easily because all application servers handle -user requests simultaneously. The database, Redis, and GitLab application are -all deployed on separate servers. The configuration is **only** highly-available -if the database, Redis and storage are also configured as such. +- Unicorn/Workhorse - Web-requests (UI, API, Git over HTTP) +- Sidekiq - Asynchronous/Background jobs +- PostgreSQL - Database + - Consul - Database service discovery and health checks/failover + - PGBouncer - Database pool manager +- Redis - Key/Value store (User sessions, cache, queue for Sidekiq) + - Sentinel - Redis health check/failover manager +- Gitaly - Provides high-level RPC access to Git repositories -Follow the steps below to configure an active/active setup: +## Scalable Architecture Examples + +When an organization reaches a certain threshold it will be necessary to scale +the GitLab instance. Still, true high availability may not be necessary. There +are options for scaling GitLab instances relatively easily without incurring the +infrastructure and maintenance costs of full high availability. + +### Basic Scaling + +This is the simplest form of scaling and will work for the majority of +cases. Backend components such as PostgreSQL, Redis and storage are offloaded +to their own nodes while the remaining GitLab components all run on 2 or more +application nodes. + +This form of scaling also works well in a cloud environment when it is more +cost-effective to deploy several small nodes rather than a single +larger one. + +- 1 PostgreSQL node +- 1 Redis node +- 2 or more GitLab application nodes (Unicorn, Workhorse, Sidekiq) +- 1 NFS/Gitaly storage server + +#### Installation Instructions + +Complete the following installation steps in order. A link at the end of each +section will bring you back to the Scalable Architecture Examples section so +you can continue with the next step. + +1. [PostgreSQL](./database.md#postgresql-in-a-scaled-environment) +1. [Redis](./redis.md#redis-in-a-scaled-environment) +1. [Gitaly](./gitaly.md) (recommended) or [NFS](./nfs.md) +1. [GitLab application nodes](./gitlab.md) + +### Full Scaling + +For very large installations it may be necessary to further split components +for maximum scalability. In a fully-scaled architecture the application node +is split into separate Sidekiq and Unicorn/Workhorse nodes. One indication that +this architecture is required is if Sidekiq queues begin to periodically increase +in size, indicating that there is contention or not enough resources. + +- 1 PostgreSQL node +- 1 Redis node +- 2 or more GitLab application nodes (Unicorn, Workhorse) +- 2 or more Sidekiq nodes +- 2 or more NFS/Gitaly storage servers + +## High Availability Architecture Examples + +When organizations require scaling *and* high availability the following +architectures can be utilized. As the introduction section at the top of this +page mentions, there is a tradeoff between cost/complexity and uptime. Be sure +this complexity is absolutely required before taking the step into full +high availability. + +For all examples below, we recommend running Consul and Redis Sentinel on +dedicated nodes. If Consul is running on PostgreSQL nodes or Sentinel on +Redis nodes there is a potential that high resource usage by PostgreSQL or +Redis could prevent communication between the other Consul and Sentinel nodes. +This may lead to the other nodes believing a failure has occurred and automated +failover is necessary. Isolating them from the services they monitor reduces +the chances of split-brain. + +The examples below do not really address high availability of NFS. Some enterprises +have access to NFS appliances that manage availability. This is the best case +scenario. In the future, GitLab may offer a more user-friendly solution to +[GitLab HA Storage](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/2472). + +There are many options in between each of these examples. Work with GitLab Support +to understand the best starting point for your workload and adapt from there. + +### Horizontal + +This is the simplest form of high availability and scaling. It requires the +fewest number of individual servers (virtual or physical) but does have some +trade-offs and limits. + +This architecture will work well for many GitLab customers. Larger customers +may begin to notice certain events cause contention/high load - for example, +cloning many large repositories with binary files, high API usage, a large +number of enqueued Sidekiq jobs, etc. If this happens you should consider +moving to a hybrid or fully distributed architecture depending on what is causing +the contention. + +- 3 PostgreSQL nodes +- 2 Redis nodes +- 3 Consul/Sentinel nodes +- 2 or more GitLab application nodes (Unicorn, Workhorse, Sidekiq, PGBouncer) +- 1 NFS/Gitaly server + + + +### Hybrid + +In this architecture, certain components are split on dedicated nodes so high +resource usage of one component does not interfere with others. In larger +environments this is a good architecture to consider if you foresee or do have +contention due to certain workloads. + +- 3 PostgreSQL nodes +- 2 Redis nodes +- 3 Consul/Sentinel nodes +- 2 or more Sidekiq nodes +- 2 or more Web nodes (Unicorn, Workhorse, PGBouncer) +- 1 or more NFS/Gitaly servers + + + +### Fully Distributed + +This architecture scales to hundreds of thousands of users and projects and is +the basis of the GitLab.com architecture. While this scales well it also comes +with the added complexity of many more nodes to configure, manage and monitor. + +- 3 PostgreSQL nodes +- 4 or more Redis nodes (2 separate clusters for persistent and cache data) +- 3 Consul nodes +- 3 Sentinel nodes +- Multiple dedicated Sidekiq nodes (Split into real-time, best effort, ASAP, + CI Pipeline and Pull Mirror sets) +- 2 or more Git nodes (Git over SSH/Git over HTTP) +- 2 or more API nodes (All requests to `/api`) +- 2 or more Web nodes (All other web requests) +- 2 or more NFS/Gitaly servers + + + +The following pages outline the steps necessary to configure each component +separately: 1. [Configure the database](database.md) 1. [Configure Redis](redis.md) 1. [Configure Redis for GitLab source installations](redis_source.md) 1. [Configure NFS](nfs.md) + 1. [NFS Client and Host setup](nfs_host_client_setup.md) 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) -### Active/Passive - -For pure high-availability/failover with no scaling you can use an -active/passive configuration. This utilizes DRBD (Distributed Replicated -Block Device) to keep all data in sync. DRBD requires a low latency link to -remain in sync. It is not advisable to attempt to run DRBD between data centers -or in different cloud availability zones. - -> **Note:** GitLab recommends against choosing this HA method because of the - complexity of managing DRBD and crafting automatic failover. This is - *compatible* with GitLab, but not officially *supported*. If you are - an EE customer, support will help you with GitLab related problems, but if the - root cause is identified as DRBD, we will not troubleshoot further. - -Components/Servers Required: 2 servers/virtual machines (one active/one passive) diff --git a/doc/administration/high_availability/alpha_database.md b/doc/administration/high_availability/alpha_database.md new file mode 100644 index 00000000000..7bf20be60e6 --- /dev/null +++ b/doc/administration/high_availability/alpha_database.md @@ -0,0 +1,6 @@ +--- +redirect_to: 'database.md' +--- + +This documentation has been moved to the main +[database documentation](database.md#configure_using_omnibus_for_high_availability). diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md new file mode 100644 index 00000000000..056b7fc15d9 --- /dev/null +++ b/doc/administration/high_availability/consul.md @@ -0,0 +1,105 @@ +# Working with the bundled Consul service **[PREMIUM ONLY]** + +## Overview + +As part of its High Availability stack, GitLab Premium includes a bundled version of [Consul](http://consul.io) that can be managed through `/etc/gitlab/gitlab.rb`. + +A Consul cluster consists of multiple server agents, as well as client agents that run on other nodes which need to talk to the consul cluster. + +## Operations + +### Checking cluster membership + +To see which nodes are part of the cluster, run the following on any member in the cluster +``` +# /opt/gitlab/embedded/bin/consul members +Node Address Status Type Build Protocol DC +consul-b XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +consul-c XX.XX.X.Y:8301 alive server 0.9.0 2 gitlab_consul +db-a XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul +db-b XX.XX.X.Y:8301 alive client 0.9.0 2 gitlab_consul +``` + +Ideally all nodes will have a `Status` of `alive`. + +### Restarting the server cluster + +**Note**: This section only applies to server agents. It is safe to restart client agents whenever needed. + +If it is necessary to restart the server cluster, it is important to do this in a controlled fashion in order to maintain quorum. If quorum is lost, you will need to follow the consul [outage recovery](#outage-recovery) process to recover the cluster. + +To be safe, we recommend you only restart one server agent at a time to ensure the cluster remains intact. + +For larger clusters, it is possible to restart multiple agents at a time. See the [Consul consensus document](https://www.consul.io/docs/internals/consensus.html#deployment-table) for how many failures it can tolerate. This will be the number of simulateneous restarts it can sustain. + +## Troubleshooting + +### Consul server agents unable to communicate + +By default, the server agents will attempt to [bind](https://www.consul.io/docs/agent/options.html#_bind) to '0.0.0.0', but they will advertise the first private IP address on the node for other agents to communicate with them. If the other nodes cannot communicate with a node on this address, then the cluster will have a failed status. + +You will see messages like the following in `gitlab-ctl tail consul` output if you are running into this issue: + +``` +2017-09-25_19:53:39.90821 2017/09/25 19:53:39 [WARN] raft: no known peers, aborting election +2017-09-25_19:53:41.74356 2017/09/25 19:53:41 [ERR] agent: failed to sync remote state: No cluster leader +``` + + +To fix this: + +1. Pick an address on each node that all of the other nodes can reach this node through. +1. Update your `/etc/gitlab/gitlab.rb` + + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` +1. Run `gitlab-ctl reconfigure` + +If you still see the errors, you may have to [erase the consul database and reinitialize](#recreate-from-scratch) on the affected node. + +### Consul agents do not start - Multiple private IPs + +In the case that a node has multiple private IPs the agent be confused as to which of the private addresses to advertise, and then immediately exit on start. + +You will see messages like the following in `gitlab-ctl tail consul` output if you are running into this issue: + +``` +2017-11-09_17:41:45.52876 ==> Starting Consul agent... +2017-11-09_17:41:45.53057 ==> Error creating agent: Failed to get advertise address: Multiple private IPs found. Please configure one. +``` + +To fix this: + +1. Pick an address on the node that all of the other nodes can reach this node through. +1. Update your `/etc/gitlab/gitlab.rb` + + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` +1. Run `gitlab-ctl reconfigure` + +### Outage recovery + +If you lost enough server agents in the cluster to break quorum, then the cluster is considered failed, and it will not function without manual intervenetion. + +#### Recreate from scratch +By default, GitLab does not store anything in the consul cluster that cannot be recreated. To erase the consul database and reinitialize + +``` +# gitlab-ctl stop consul +# rm -rf /var/opt/gitlab/consul/data +# gitlab-ctl start consul +``` + +After this, the cluster should start back up, and the server agents rejoin. Shortly after that, the client agents should rejoin as well. + +#### Recover a failed cluster +If you have taken advantage of consul to store other data, and want to restore the failed cluster, please follow the [Consul guide](https://www.consul.io/docs/guides/outage.html) to recover a failed cluster. diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index c1eeb40b98f..1648b6b848a 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -1,11 +1,6 @@ -# Configuring a Database for GitLab HA +# Configuring PostgreSQL for Scaling and High Availability -You can choose to install and manage a database server (PostgreSQL/MySQL) -yourself, or you can use GitLab Omnibus packages to help. GitLab recommends -PostgreSQL. This is the database that will be installed if you use the -Omnibus package to manage your database. - -## Configure your own database server +## Provide your own PostgreSQL instance **[CORE ONLY]** If you're hosting GitLab on a cloud provider, you can optionally use a managed service for PostgreSQL. For example, AWS offers a managed Relational @@ -20,91 +15,1147 @@ If you use a cloud-managed service, or provide your own PostgreSQL: 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring GitLab for HA](gitlab.md). -## Configure using Omnibus +## PostgreSQL in a Scaled Environment -1. Download/install GitLab Omnibus using **steps 1 and 2** from - [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other - steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. - Be sure to change the `external_url` to match your eventual GitLab front-end - URL. If there is a directive listed below that you do not see in the configuration, be sure to add it. +This section is relevant for [Scaled Architecture](./README.md#scalable-architecture-examples) +environments including [Basic Scaling](./README.md#basic-scaling) and +[Full Scaling](./README.md#full-scaling). - ```ruby - external_url 'https://gitlab.example.com' +### Provide your own PostgreSQL instance **[CORE ONLY]** + +If you want to use your own deployed PostgreSQL instance(s), +see [Provide your own PostgreSQL instance](#provide-your-own-postgresql-instance-core-only) +for more details. However, you can use the GitLab Omnibus package to easily +deploy the bundled PostgreSQL. + +### Standalone PostgreSQL using GitLab Omnibus **[CORE ONLY]** + +1. SSH into the PostgreSQL server. +1. [Download/install](https://about.gitlab.com/installation) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. +1. Generate a password hash for PostgreSQL. This assumes you will use the default + username of `gitlab` (recommended). The command will request a password + and confirmation. Use the value that is output by this command in the next + step as the value of `POSTGRESQL_PASSWORD_HASH`. + ```sh + sudo gitlab-ctl pg-password-md5 gitlab + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder + values appropriately. + + - `POSTGRESQL_PASSWORD_HASH` - The value output from the previous step + - `APPLICATION_SERVER_IP_BLOCKS` - A space delimited list of IP subnets or IP + addresses of the GitLab application servers that will connect to the + database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` + + ```ruby # Disable all components except PostgreSQL roles ['postgres_role'] + repmgr['enable'] = false + consul['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_monitor['enable'] = false + + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + # ???? + postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + ``` + + NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 + +1. [Reconfigure GitLab] for the changes to take effect. +1. Note the PostgreSQL node's IP address or hostname, port, and + plain text password. These will be necessary when configuring the GitLab + application servers later. + +Advanced configuration options are supported and can be added if +needed. + +Continue configuration of other components by going +[back to Scaled Architectures](./README.md#scalable-architecture-examples) + +## PostgreSQL with High Availability + +This section is relevant for [High Availability Architecture](./README.md#high-availability-architecture-examples) +environments including [Horizontal](./README.md#horizontal), +[Hybrid](./README.md#hybrid), and +[Fully Distributed](./README.md#fully-distributed). + +### Provide your own PostgreSQL instance **[CORE ONLY]** + +If you want to use your own deployed PostgreSQL instance(s), +see [Provide your own PostgreSQL instance](#provide-your-own-postgresql-instance-core-only) +for more details. However, you can use the GitLab Omnibus package to easily +deploy the bundled PostgreSQL. + +### High Availability with GitLab Omnibus **[PREMIUM ONLY]** + +> Important notes: +> - This document will focus only on configuration supported with [GitLab Premium](https://about.gitlab.com/pricing/), using the Omnibus GitLab package. +> - If you are a Community Edition or Starter user, consider using a cloud hosted solution. +> - This document will not cover installations from source. +> +> - If HA setup is not what you were looking for, see the [database configuration document](http://docs.gitlab.com/omnibus/settings/database.html) +> for the Omnibus GitLab packages. + +> Please read this document fully before attempting to configure PostgreSQL HA +> for GitLab. +> +> This configuration is GA in EE 10.2. + +The recommended configuration for a PostgreSQL HA requires: + +- A minimum of three database nodes + - Each node will run the following services: + - `PostgreSQL` - The database itself + - `repmgrd` - A service to monitor, and handle failover in case of a failure + - `Consul` agent - Used for service discovery, to alert other nodes when failover occurs +- A minimum of three `Consul` server nodes +- A minimum of one `pgbouncer` service node + +You also need to take into consideration the underlying network topology, +making sure you have redundant connectivity between all Database and GitLab instances, +otherwise the networks will become a single point of failure. + +#### Architecture + + + +Database nodes run two services with PostgreSQL: + +- Repmgrd. Monitors the cluster and handles failover when issues with the master occur. The failover consists of: + - Selecting a new master for the cluster. + - Promoting the new node to master. + - Instructing remaining servers to follow the new master node. + + On failure, the old master node is automatically evicted from the cluster, and should be rejoined manually once recovered. +- Consul. Monitors the status of each node in the database cluster and tracks its health in a service definition on the consul cluster. + +Alongside pgbouncer, there is a consul agent that watches the status of the PostgreSQL service. If that status changes, consul runs a script which updates the configuration and reloads pgbouncer + +##### Connection flow + +Each service in the package comes with a set of [default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#ports). You may need to make specific firewall rules for the connections listed below: + +- Application servers connect to [PgBouncer default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#pgbouncer) +- PgBouncer connects to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- Repmgr connects to the database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- Postgres secondaries connect to the primary database servers [PostgreSQL default port](https://docs.gitlab.com/omnibus/package-information/defaults.html#postgresql) +- Consul servers and agents connect to each others [Consul default ports](https://docs.gitlab.com/omnibus/package-information/defaults.html#consul) + +#### Required information + +Before proceeding with configuration, you will need to collect all the necessary +information. + +##### Network information + +PostgreSQL does not listen on any network interface by default. It needs to know +which IP address to listen on in order to be accessible to other services. +Similarly, PostgreSQL access is controlled based on the network source. + +This is why you will need: + +> IP address of each nodes network interface +> - This can be set to `0.0.0.0` to listen on all interfaces. It cannot +> be set to the loopack address `127.0.0.1` +> +> Network Address +> - This can be in subnet (i.e. `192.168.0.0/255.255.255.0`) or CIDR (i.e. +> `192.168.0.0/24`) form. + +##### User information + +Various services require different configuration to secure +the communication as well as information required for running the service. +Bellow you will find details on each service and the minimum required +information you need to provide. + +##### Consul information + +When using default setup, minimum configuration requires: + +- `CONSUL_USERNAME`. Defaults to `gitlab-consul` +- `CONSUL_DATABASE_PASSWORD`. Password for the database user. +- `CONSUL_PASSWORD_HASH`. This is a hash generated out of consul username/password pair. + Can be generated with: + + ```sh + sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME + ``` + +- `CONSUL_SERVER_NODES`. The IP addresses or DNS records of the Consul server nodes. + +Few notes on the service itself: + +- The service runs under a system account, by default `gitlab-consul`. + - If you are using a different username, you will have to specify it. We +will refer to it with `CONSUL_USERNAME`, +- There will be a database user created with read only access to the repmgr +database +- Passwords will be stored in the following locations: + - `/etc/gitlab/gitlab.rb`: hashed + - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed + - `/var/opt/gitlab/gitlab-consul/.pgpass`: plaintext + +##### PostgreSQL information + +When configuring PostgreSQL, we will set `max_wal_senders` to one more than +the number of database nodes in the cluster. +This is used to prevent replication from using up all of the +available database connections. + +> Note: +> - In this document we are assuming 3 database nodes, which makes this configuration: + +``` +postgresql['max_wal_senders'] = 4 +``` + +As previously mentioned, you'll have to prepare the network subnets that will +be allowed to authenticate with the database. +You'll also need to supply the IP addresses or DNS records of Consul +server nodes. + +We will need the following password information for the application's database user: + +- `POSTGRESQL_USERNAME`. Defaults to `gitlab` +- `POSTGRESQL_USER_PASSWORD`. The password for the database user +- `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. + Can be generated with: + + ```sh + sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME + ``` + +##### Pgbouncer information + +When using default setup, minimum configuration requires: + +- `PGBOUNCER_USERNAME`. Defaults to `pgbouncer` +- `PGBOUNCER_PASSWORD`. This is a password for pgbouncer service. +- `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of pgbouncer username/password pair. + Can be generated with: + + ```sh + sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME + ``` + +- `PGBOUNCER_NODE`, is the IP address or a FQDN of the node running Pgbouncer. + +Few notes on the service itself: + +- The service runs as the same system account as the database + - In the package, this is by default `gitlab-psql` +- If you use a non-default user account for Pgbouncer service (by default `pgbouncer`), you will have to specify this username. We will refer to this requirement with `PGBOUNCER_USERNAME`. +- The service will have a regular database user account generated for it + - This defaults to `repmgr` +- Passwords will be stored in the following locations: + - `/etc/gitlab/gitlab.rb`: hashed, and in plain text + - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed + +##### Repmgr information + +When using default setup, you will only have to prepare the network subnets that will +be allowed to authenticate with the service. + +Few notes on the service itself: + +- The service runs under the same system account as the database + - In the package, this is by default `gitlab-psql` +- The service will have a superuser database user account generated for it + - This defaults to `gitlab_repmgr` + +#### Installing Omnibus GitLab + +First, make sure to [download/install](https://about.gitlab.com/installation) +GitLab Omnibus **on each node**. + +Make sure you install the necessary dependencies from step 1, +add GitLab package repository from step 2. +When installing the GitLab package, do not supply `EXTERNAL_URL` value. + +#### Configuring the Consul nodes + +On each Consul node perform the following: + +1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information) before executing the next step. + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except Consul + roles ['consul_role'] + + # START user configuration + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + server: true, + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + # + # END user configuration + ``` + + > `consul_role` was introduced with GitLab 10.3 + +1. [Reconfigure GitLab] for the changes to take effect. + +##### Consul Checkpoint + +Before moving on, make sure Consul is configured correctly. Run the following +command to verify all server nodes are communicating: + +``` +/opt/gitlab/embedded/bin/consul members +``` + +The output should be similar to: + +``` +Node Address Status Type Build Protocol DC +CONSUL_NODE_ONE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul +CONSUL_NODE_TWO XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul +CONSUL_NODE_THREE XXX.XXX.XXX.YYY:8301 alive server 0.9.2 2 gitlab_consul +``` + +If any of the nodes isn't `alive` or if any of the three nodes are missing, +check the [Troubleshooting section](#troubleshooting) before proceeding. + +#### Configuring the Database nodes + +1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information), [`POSTGRESQL_PASSWORD_HASH`](#postgresql-information), the [number of db nodes](#postgresql-information), and the [network address](#network-information) before executing the next step. + +1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] # PostgreSQL configuration - gitlab_rails['db_password'] = 'DB password' - postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'] postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' # Disable automatic database migrations gitlab_rails['auto_migrate'] = false + + # Configure the consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + # Replace X with value of number of db nodes + 1 + postgresql['max_wal_senders'] = X + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) + + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration ``` -1. Run `sudo gitlab-ctl reconfigure` to install and configure PostgreSQL. + > `postgres_role` was introduced with GitLab 10.3 + +1. On secondary nodes, add all the configuration specified above for primary node + to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration + to inform gitlab-ctl that they are standby nodes initially and it need not + attempt to register them as primary node + ``` + # HA setting to specify if a node should attempt to be master on initialization + repmgr['master_on_initialization'] = false + ``` - > **Note**: This `reconfigure` step will result in some errors. - That's OK - don't be alarmed. +1. [Reconfigure GitLab] for te changes to take effect. + +> Please note: +> - If you want your database to listen on a specific interface, change the config: +> `postgresql['listen_address'] = '0.0.0.0'` +> - If your Pgbouncer service runs under a different user account, +> you also need to specify: `postgresql['pgbouncer_user'] = PGBOUNCER_USERNAME` in +> your configuration + +##### Database nodes post-configuration + +###### Primary node + +Select one node as a primary node. 1. Open a database prompt: + ```sh + gitlab-psql -d gitlabhq_production ``` - su - gitlab-psql - /bin/bash - psql -h /var/opt/gitlab/postgresql -d template1 - # Output: +1. Enable the `pg_trgm` extension: - psql (9.2.15) - Type "help" for help. + ```sh + CREATE EXTENSION pg_trgm; + ``` + +1. Exit the database prompt by typing `\q` and Enter. + +1. Verify the cluster is initialized with one node: - template1=# + ```sh + gitlab-ctl repmgr cluster show ``` -1. Run the following command at the database prompt and you will be asked to - enter the new password for the PostgreSQL superuser. + The output should be similar to the following: ``` - \password + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` - # Output: +1. Note down the hostname/ip in the connection string: `host=HOSTNAME`. We will + refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value + is not an IP address, it will need to be a resolvable name (via DNS or + `/etc/hosts`) - Enter new password: - Enter it again: + +###### Secondary nodes + +1. Set up the repmgr standby: + + ```sh + gitlab-ctl repmgr standby setup MASTER_NODE_NAME ``` -1. Similarly, set the password for the `gitlab` database user. Use the same - password that you specified in the `/etc/gitlab/gitlab.rb` file for - `gitlab_rails['db_password']`. + Do note that this will remove the existing data on the node. The command + has a wait time. + The output should be similar to the following: + + ```console + # gitlab-ctl repmgr standby setup MASTER_NODE_NAME + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s ``` - \password gitlab - # Output: +1. Verify the node now appears in the cluster: - Enter new password: - Enter it again: + ```sh + gitlab-ctl repmgr cluster show ``` -1. Exit from editing `template1` prompt by typing `\q` and Enter. -1. Enable the `pg_trgm` extension within the `gitlabhq_production` database: - + + The output should be similar to the following: + + ``` + Role | Name | Upstream | Connection String + ----------+---------|-----------|------------------------------------------------ + * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` + +Repeat the above steps on all secondary nodes. + +##### Database checkpoint + +Before moving on, make sure the databases are configured correctly. Run the +following command on the **primary** node to verify that replication is working +properly: + +``` +gitlab-ctl repmgr cluster show +``` + +The output should be similar to: + +``` +Role | Name | Upstream | Connection String +----------+--------------|--------------|-------------------------------------------------------------------- +* master | MASTER | | host=MASTER port=5432 user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY port=5432 user=gitlab_repmgr dbname=gitlab_repmgr +``` + +If the 'Role' column for any node says "FAILED", check the +[Troubleshooting section](#troubleshooting) before proceeding. + +Also, check that the check master command works successfully on each node: + +``` +su - gitlab-consul +gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' +``` + +This command relies on exit codes to tell Consul whether a particular node is a master +or secondary. The most important thing here is that this command does not produce errors. +If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. +Check the [Troubleshooting section](#troubleshooting) before proceeding. + +#### Configuring the Pgbouncer node + +1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`CONSUL_PASSWORD_HASH`](#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information) before executing the next step. + +1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: + + ```ruby + # Disable all components except Pgbouncer and Consul agent + roles ['pgbouncer_role'] + + # Configure Pgbouncer + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + + # Configure Consul agent + consul['watchers'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # Replace CONSUL_PASSWORD_HASH with with a generated md5 value + # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value + pgbouncer['users'] = { + 'gitlab-consul': { + password: 'CONSUL_PASSWORD_HASH' + }, + 'pgbouncer': { + password: 'PGBOUNCER_PASSWORD_HASH' + } + } + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration ``` + + > `pgbouncer_role` was introduced with GitLab 10.3 + +1. [Reconfigure GitLab] for the changes to take effect. + +1. Create a `.pgpass` file so Consule is able to + reload pgbouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked: + + ```sh + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + +##### PGBouncer Checkpoint + +1. Ensure the node is talking to the current master: + + ```sh + gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + ``` + + If there is an error `psql: ERROR: Auth failed` after typing in the + password, ensure you previously generated the MD5 password hashes with the correct + format. The correct format is to concatenate the password and the username: + `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text + needed to generate an MD5 password hash for the `pgbouncer` user. + +1. Once the console prompt is available, run the following queries: + + ```sh + show databases ; show clients ; + ``` + + The output should be similar to the following: + + ``` + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections + ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production | | 20 | 0 | | 0 | 0 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 + (2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | remote_pid | tls + ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+----- + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 56846 | 127.0.0.1 | 6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 | | 0 | + (2 rows) + ``` + +#### Configuring the Application nodes + +These will be the nodes running the `gitlab-rails` service. You may have other +attributes set, but the following need to be set. + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Disable PostgreSQL on the application node + postgresql['enable'] = false + + gitlab_rails['db_host'] = 'PGBOUNCER_NODE' + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure GitLab] for the changes to take effect. + +##### Application node post-configuration + +Ensure that all migrations ran: + +```sh +gitlab-rake gitlab:db:configure +``` + +> **Note**: If you encounter a `rake aborted!` error stating that PGBouncer is failing to connect to +PostgreSQL it may be that your PGBouncer node's IP address is missing from +PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See +[PGBouncer error `ERROR: pgbouncer cannot connect to server`](#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +in the Troubleshooting section before proceeding. + +##### Ensure GitLab is running + +At this point, your GitLab instance should be up and running. Verify you are +able to login, and create issues and merge requests. If you have troubles check +the [Troubleshooting section](#troubleshooting). + +#### Example configuration + +Here we'll show you some fully expanded example configurations. + +##### Example recommended setup + +This example uses 3 consul servers, 3 postgresql servers, and 1 application node. + +We start with all servers on the same 10.6.0.0/16 private network range, they +can connect to each freely other on those addresses. + +Here is a list and description of each machine and the assigned IP: + +* `10.6.0.11`: Consul 1 +* `10.6.0.12`: Consul 2 +* `10.6.0.13`: Consul 3 +* `10.6.0.21`: PostgreSQL master +* `10.6.0.22`: PostgreSQL secondary +* `10.6.0.23`: PostgreSQL secondary +* `10.6.0.31`: GitLab application + +All passwords are set to `toomanysecrets`, please do not use this password or derived hashes. + +The external_url for GitLab is `http://gitlab.example.com` + +Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. + +##### Example recommended setup for Consul servers + +On each server edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except Consul +roles ['consul_role'] + +consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +##### Example recommended setup for PostgreSQL servers + +###### Primary node + +On primary node edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except PostgreSQL and Repmgr and Consul +roles ['postgres_role'] + +# PostgreSQL configuration +postgresql['listen_address'] = '0.0.0.0' +postgresql['hot_standby'] = 'on' +postgresql['wal_level'] = 'replica' +postgresql['shared_preload_libraries'] = 'repmgr_funcs' + +# Disable automatic database migrations +gitlab_rails['auto_migrate'] = false + +# Configure the consul agent +consul['services'] = %w(postgresql) + +postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' +postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' +postgresql['max_wal_senders'] = 4 + +postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) +repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) + +consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +###### Secondary nodes + +On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the configuration +added to primary node, noted above. In addition, append the following +configuration + +``` +# HA setting to specify if a node should attempt to be master on initialization +repmgr['master_on_initialization'] = false +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +##### Example recommended setup for application server + +On the server edit `/etc/gitlab/gitlab.rb`: + +```ruby +external_url 'http://gitlab.example.com' + +gitlab_rails['db_host'] = '127.0.0.1' +gitlab_rails['db_port'] = 6432 +gitlab_rails['db_password'] = 'toomanysecrets' +gitlab_rails['auto_migrate'] = false + +postgresql['enable'] = false +pgbouncer['enable'] = true +consul['enable'] = true + +# Configure Pgbouncer +pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + +# Configure Consul agent +consul['watchers'] = %w(postgresql) + +pgbouncer['users'] = { + 'gitlab-consul': { + password: '5e0e3263571e3704ad655076301d6ebe' + }, + 'pgbouncer': { + password: '771a8625958a529132abe6f1a4acb19c' + } +} + +consul['configuration'] = { + retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) +} +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +##### Example recommended setup manual steps + +After deploying the configuration follow these steps: + +1. On `10.6.0.21`, our primary database + + Enable the `pg_trgm` extension + + ```sh gitlab-psql -d gitlabhq_production - + ``` + + ``` CREATE EXTENSION pg_trgm; + ``` + +1. On `10.6.0.22`, our first standby database - # Output: + Make this node a standby of the primary - CREATE EXTENSION + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 ``` -1. Exit the database prompt by typing `\q` and Enter. -1. Exit the `gitlab-psql` user by running `exit` twice. -1. Run `sudo gitlab-ctl reconfigure` a final time. -1. Configure the GitLab application servers with the appropriate details. - This step is covered in [Configuring GitLab for HA](gitlab.md). + +1. On `10.6.0.23`, our second standby database + + Make this node a standby of the primary + + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` + +1. On `10.6.0.31`, our application server + + Set gitlab-consul's pgbouncer password to `toomanysecrets` + + ```sh + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` + + Run database migrations + + ```sh + gitlab-rake gitlab:db:configure + ``` + +#### Example minimal setup + +This example uses 3 postgresql servers, and 1 application node. + +It differs from the [recommended setup](#example-recommended-setup) by moving the consul servers into the same servers we use for PostgreSQL. +The trade-off is between reducing server counts, against the increased operational complexity of needing to deal with postgres [failover](#failover-procedure) and [restore](#restore-procedure) procedures in addition to [consul outage recovery](consul.md#outage-recovery) on the same set of machines. + +In this example we start with all servers on the same 10.6.0.0/16 private network range, they can connect to each freely other on those addresses. + +Here is a list and description of each machine and the assigned IP: + +* `10.6.0.21`: PostgreSQL master +* `10.6.0.22`: PostgreSQL secondary +* `10.6.0.23`: PostgreSQL secondary +* `10.6.0.31`: GitLab application + +All passwords are set to `toomanysecrets`, please do not use this password or derived hashes. + +The external_url for GitLab is `http://gitlab.example.com` + +Please note that after the initial configuration, if a failover occurs, the PostgresSQL master will change to one of the available secondaries until it is failed back. + +##### Example minimal configuration for database servers + +##### Primary node +On primary database node edit `/etc/gitlab/gitlab.rb`: + +```ruby +# Disable all components except PostgreSQL, Repmgr, and Consul +roles ['postgres_role'] + +# PostgreSQL configuration +postgresql['listen_address'] = '0.0.0.0' +postgresql['hot_standby'] = 'on' +postgresql['wal_level'] = 'replica' +postgresql['shared_preload_libraries'] = 'repmgr_funcs' + +# Disable automatic database migrations +gitlab_rails['auto_migrate'] = false + +# Configure the consul agent +consul['services'] = %w(postgresql) + +postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' +postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' +postgresql['max_wal_senders'] = 4 + +postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) +repmgr['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) + +consul['configuration'] = { + server: true, + retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) +} +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +###### Secondary nodes + +On secondary nodes, edit `/etc/gitlab/gitlab.rb` and add all the information added +to primary node, noted above. In addition, append the following configuration + +``` +# HA setting to specify if a node should attempt to be master on initialization +repmgr['master_on_initialization'] = false +``` + +##### Example minimal configuration for application server + +On the server edit `/etc/gitlab/gitlab.rb`: + +```ruby +external_url 'http://gitlab.example.com' + +gitlab_rails['db_host'] = '127.0.0.1' +gitlab_rails['db_port'] = 6432 +gitlab_rails['db_password'] = 'toomanysecrets' +gitlab_rails['auto_migrate'] = false + +postgresql['enable'] = false +pgbouncer['enable'] = true +consul['enable'] = true + +# Configure Pgbouncer +pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) + +# Configure Consul agent +consul['watchers'] = %w(postgresql) + +pgbouncer['users'] = { + 'gitlab-consul': { + password: '5e0e3263571e3704ad655076301d6ebe' + }, + 'pgbouncer': { + password: '771a8625958a529132abe6f1a4acb19c' + } +} + +consul['configuration'] = { + retry_join: %w(10.6.0.21 10.6.0.22 10.6.0.23) +} +``` + +[Reconfigure Omnibus GitLab][reconfigure GitLab] for the changes to take effect. + +##### Example minimal setup manual steps + +The manual steps for this configuration are the same as for the [example recommended setup](#example-recommended-setup-manual-steps). + +#### Failover procedure + +By default, if the master database fails, `repmgrd` should promote one of the +standby nodes to master automatically, and consul will update pgbouncer with +the new master. + +If you need to failover manually, you have two options: + +**Shutdown the current master database** + +Run: + +```sh +gitlab-ctl stop postgresql +``` + +The automated failover process will see this and failover to one of the +standby nodes. + +**Or perform a manual failover** + +1. Ensure the old master node is not still active. +1. Login to the server that should become the new master and run: + + ```sh + gitlab-ctl repmgr standby promote + ``` + +1. If there are any other standby servers in the cluster, have them follow + the new master server: + + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + ``` + +#### Restore procedure + +If a node fails, it can be removed from the cluster, or added back as a standby +after it has been restored to service. + +- If you want to remove the node from the cluster, on any other node in the + cluster, run: + + ```sh + gitlab-ctl repmgr standby unregister --node=X + ``` + + where X is the value of node in `repmgr.conf` on the old server. + + To find this, you can use: + + ```sh + awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf + ``` + + It will output something like: + + ``` + 959789412 + ``` + + Then you will use this id to unregister the node: + + ```sh + gitlab-ctl repmgr standby unregister --node=959789412 + ``` + +- To add the node as a standby server: + + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + gitlab-ctl restart repmgrd + ``` + + CAUTION: **Warning:** When the server is brought back online, and before + you switch it to a standby node, repmgr will report that there are two masters. + If there are any clients that are still attempting to write to the old master, + this will cause a split, and the old master will need to be resynced from + scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. + +#### Alternate configurations + +##### Database authorization + +By default, we give any host on the database network the permission to perform +repmgr operations using PostgreSQL's `trust` method. If you do not want this +level of trust, there are alternatives. + +You can trust only the specific nodes that will be database clusters, or you +can require md5 authentication. + +##### Trust specific addresses + +If you know the IP address, or FQDN of all database and pgbouncer nodes in the +cluster, you can trust only those nodes. + +In `/etc/gitlab/gitlab.rb` on all of the database nodes, set +`repmgr['trust_auth_cidr_addresses']` to an array of strings containing all of +the addresses. + +If setting to a node's FQDN, they must have a corresponding PTR record in DNS. +If setting to a node's IP address, specify it as `XXX.XXX.XXX.XXX/32`. + +For example: + +```ruby +repmgr['trust_auth_cidr_addresses'] = %w(192.168.1.44/32 db2.example.com) +``` + + +##### MD5 Authentication + +If you are running on an untrusted network, repmgr can use md5 authentication +with a [.pgpass file](https://www.postgresql.org/docs/9.6/static/libpq-pgpass.html) +to authenticate. + +You can specify by IP address, FQDN, or by subnet, using the same format as in +the previous section: + +1. On the current master node, create a password for the `gitlab` and + `gitlab_repmgr` user: + + ```sh + gitlab-psql -d template1 + template1=# \password gitlab_repmgr + Enter password: **** + Confirm password: **** + template1=# \password gitlab + ``` + +1. On each database node: + + 1. Edit `/etc/gitlab/gitlab.rb`: + 1. Ensure `repmgr['trust_auth_cidr_addresses']` is **not** set + 1. Set `postgresql['md5_auth_cidr_addresses']` to the desired value + 1. Set `postgresql['sql_replication_user'] = 'gitlab_repmgr'` + 1. Reconfigure with `gitlab-ctl reconfigure` + 1. Restart postgresql with `gitlab-ctl restart postgresql` + + 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to + when asked: + + ```sh + gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' + ``` + +1. On each pgbouncer node, edit `/etc/gitlab/gitlab.rb`: + 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for + the `gitlab` database user + 1. [Reconfigure GitLab] for the changes to take effect + +## Troubleshooting + +#### Consul and PostgreSQL changes not taking effect. + +Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. + +To restart either service, run `gitlab-ctl restart SERVICE` + +For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. + +On the consul server nodes, it is important to restart the consul service in a controlled fashion. Read our [consul documentation](consul.md#restarting-the-server-cluster) for instructions on how to restart the service. + +#### `gitlab-ctl repmgr-check-master` command produces errors + +If this command displays errors about database permissions it is likely that something failed during +install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these +steps to fix the problem: + +1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` +1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` +1. Exit the database prompt - `\q` +1. [Reconfigure GitLab] and the user will be re-added with the proper permissions. +1. Change to the `gitlab-consul` user - `su - gitlab-consul` +1. Try the check command again - `gitlab-ctl repmgr-check-master`. + +Now there should not be errors. If errors still occur then there is another problem. + +#### PGBouncer error `ERROR: pgbouncer cannot connect to server` + +You may get this error when running `gitlab-rake gitlab:db:configure` or you +may see the error in the PGBouncer log file. + +``` +PG::ConnectionBad: ERROR: pgbouncer cannot connect to server +``` + +The problem may be that your PGBouncer node's IP address is not included in the +`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes. + +You can confirm that this is the issue by checking the PostgreSQL log on the master +database node. If you see the following error then `trust_auth_cidr_addresses` +is the problem. + +``` +2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off +``` + +To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`. + +``` +postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) +``` + +[Reconfigure GitLab] for the changes to take effect. + +#### Issues with other components + +If you're running into an issue with a component not outlined here, be sure to check the troubleshooting section of their specific documentation page. + +- [Consul](consul.md#troubleshooting) +- [PostgreSQL](http://docs.gitlab.com/omnibus/settings/database.html#troubleshooting) +- [GitLab application](gitlab.md#troubleshooting) + +## Configure using Omnibus + +**Note**: We recommend that you follow the instructions here for a full [PostgreSQL cluster](#high-availability-with-gitlab-omnibus-premium-only). +If you are reading this section due to an old bookmark, you can find that old documentation [in the repository](https://gitlab.com/gitlab-org/gitlab-ce/blob/v10.1.4/doc/administration/high_availability/database.md#configure-using-omnibus). --- @@ -114,3 +1165,6 @@ Read more on high-availability configuration: 1. [Configure NFS](nfs.md) 1. [Configure the GitLab application servers](gitlab.md) 1. [Configure the load balancers](load_balancer.md) +1. [Manage the bundled Consul cluster](consul.md) + +[reconfigure GitLab]: ../restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md new file mode 100644 index 00000000000..d44744f2af8 --- /dev/null +++ b/doc/administration/high_availability/gitaly.md @@ -0,0 +1,90 @@ +# Configuring Gitaly for Scaled and High Availability + +Gitaly does not yet support full high availability. However, Gitaly is quite +stable and is in use on GitLab.com. Scaled and highly available GitLab environments +should consider using Gitaly on a separate node. + +See the [Gitaly HA Epic](https://gitlab.com/groups/gitlab-org/-/epics/289) to +track plans and progress toward high availability support. + +This document is relevant for [Scaled Architecture](./README.md#scalable-architecture-examples) +environments and [High Availability Architecture](./README.md#high-availability-architecture-examples). + +## Running Gitaly on its own server + +Starting with GitLab 11.4, Gitaly is a replacement for NFS except +when the [Elastic Search indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer) +is used. + +NOTE: **Note:** While Gitaly can be used as a replacement for NFS, we do not recommend using EFS as it may impact GitLab's performance. Please review the [relevant documentation](nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. + +NOTE: **Note:** Gitaly network traffic is unencrypted so we recommend a firewall to +restrict access to your Gitaly server. + +The steps below are the minimum necessary to configure a Gitaly server with +Omnibus: + +1. SSH into the Gitaly server. +1. [Download/install](https://about.gitlab.com/installation) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + Gitaly must trigger some callbacks to GitLab via GitLab Shell. As a result, + the GitLab Shell secret must be the same between the other GitLab servers and + the Gitaly server. The easiest way to accomplish this is to copy `/etc/gitlab/gitlab-secrets.json` + from an existing GitLab server to the Gitaly server. Without this shared secret, + Git operations in GitLab will result in an API error. + + > **NOTE:** In most or all cases the storage paths below end in `repositories` which is + different than `path` in `git_data_dirs` of Omnibus installations. Check the + directory layout on your Gitaly server to be sure. + + ```ruby + # Enable Gitaly + gitaly['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + unicorn['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_monitor['enable'] = false + + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + gitaly['listen_addr'] = "0.0.0.0:8075" + gitaly['auth_token'] = 'abc123secret' + + gitaly['storage'] = [ + { 'name' => 'default', 'path' => '/mnt/gitlab/default/repositories' }, + { 'name' => 'storage1', 'path' => '/mnt/gitlab/storage1/repositories' }, + ] + + # To use tls for gitaly you need to add + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "path/to/cert.pem" + gitaly['key_path'] = "path/to/key.pem" + ``` + +Again, reconfigure (Omnibus) or restart (source). + +Continue configuration of other components by going back to: + +- [Scaled Architectures](./README.md#scalable-architecture-examples) +- [High Availability Architectures](./README.md#high-availability-architecture-examples) diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index d95c3acec54..888426ece5c 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -1,8 +1,4 @@ -# Configuring GitLab for HA - -Assuming you have already configured a [database](database.md), [Redis](redis.md), and [NFS](nfs.md), you can -configure the GitLab application server(s) now. Complete the steps below -for each GitLab application server in your environment. +# Configuring GitLab Scaling and High Availability > **Note:** There is some additional configuration near the bottom for additional GitLab application servers. It's important to read and understand diff --git a/doc/administration/high_availability/nfs_host_client_setup.md b/doc/administration/high_availability/nfs_host_client_setup.md new file mode 100644 index 00000000000..a8bc101dee6 --- /dev/null +++ b/doc/administration/high_availability/nfs_host_client_setup.md @@ -0,0 +1,135 @@ +# Configuring NFS for GitLab HA + +Setting up NFS for a GitLab HA setup allows all applications nodes in a cluster +to share the same files and maintain data consistency. Application nodes in an HA +setup act as clients while the NFS server plays host. + +> Note: The instructions provided in this documentation allow for setting a quick +proof of concept but will leave NFS as potential single point of failure and +therefore not recommended for use in production. Explore options such as [Pacemaker +and Corosync](http://clusterlabs.org/) for highly available NFS in production. + +Below are instructions for setting up an application node(client) in an HA cluster +to read from and write to a central NFS server(host). + +NOTE: **Note:** +Using EFS may negatively impact performance. Please review the [relevant documentation](nfs.md#avoid-using-awss-elastic-file-system-efs) for additional details. + +## NFS Server Setup + +> Follow the instructions below to set up and configure your NFS server. + +### Step 1 - Install NFS Server on Host + +Installing the nfs-kernel-server package allows you to share directories with the clients running the GitLab application. + +```sh +apt-get update +apt-get install nfs-kernel-server +``` + +### Step 2 - Export Host's Home Directory to Client + +In this setup we will share the home directory on the host with the client. Edit the exports file as below to share the host's home directory with the client. If you have multiple clients running GitLab you must enter the client IP addresses in line in the `/etc/exports` file. + +```text +#/etc/exports for one client +/home <client-ip-address>(rw,sync,no_root_squash,no_subtree_check) + +#/etc/exports for three clients +/home <client-ip-address>(rw,sync,no_root_squash,no_subtree_check) <client-2-ip-address>(rw,sync,no_root_squash,no_subtree_check) <client-3-ip-address>(rw,sync,no_root_squash,no_subtree_check) +``` + +Restart the NFS server after making changes to the `exports` file for the changes +to take effect. + +```sh +systemctl restart nfs-kernel-server +``` + +NOTE: **Note:** +You may need to update your server's firewall. See the [firewall section](#nfs-in-a-firewalled-environment) at the end of this guide. + +## Client/ GitLab application node Setup + +> Follow the instructions below to connect any GitLab rails application node running +inside your HA environment to the NFS server configured above. + +### Step 1 - Install NFS Common on Client + +The nfs-common provides NFS functionality without installing server components which +we don't need running on the application nodes. + +```sh +apt-get update +apt-get install nfs-common +``` + +### Step 2 - Create Mount Points on Client + +Create a directroy on the client that we can mount the shared directory from the host. +Please note that if your mount point directory contains any files they will be hidden +once the remote shares are mounted. An empty/new directory on the client is recommended +for this purpose. + +```sh +mkdir -p /nfs/home +``` + +Confirm that the mount point works by mounting it on the client and checking that +it is mounted with the command below: + +```sh +mount <host_ip_address>:/home +df -h +``` + +### Step 3 - Set up Automatic Mounts on Boot + +Edit `/etc/fstab` on client as below to mount the remote shares automatically at boot. +Note that GitLab requires advisory file locking, which is only supported natively in +NFS version 4. NFSv3 also supports locking as long as Linux Kernel 2.6.5+ is used. +We recommend using version 4 and do not specifically test NFSv3. + +```text +#/etc/fstab +165.227.159.85:/home /nfs/home nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 +``` + +Reboot the client and confirm that the mount point is mounted automatically. + +### Step 4 - Set up GitLab to Use NFS mounts + +When using the default Omnibus configuration you will need to share 5 data locations +between all GitLab cluster nodes. No other locations should be shared. Changing the +default file locations in `gitlab.rb` on the client allows you to have one main mount +point and have all the required locations as subdirectories to use the NFS mount for +git-data. + +```text +git_data_dirs({"default" => {"path" => "/nfs/home/var/opt/gitlab-data/git-data"}}) +gitlab_rails['uploads_directory'] = '/nfs/home/var/opt/gitlab-data/uploads' +gitlab_rails['shared_path'] = '/nfs/home/var/opt/gitlab-data/shared' +gitlab_ci['builds_directory'] = '/nfs/home/var/opt/gitlab-data/builds' +``` + +Save the changes in `gitlab.rb` and run `gitlab-ctl reconfigure`. + +## NFS in a Firewalled Environment + +If the traffic between your NFS server and NFS client(s) is subject to port filtering +by a firewall, then you will need to reconfigure that firewall to allow NFS communication. + +[This guide from TDLP](http://tldp.org/HOWTO/NFS-HOWTO/security.html#FIREWALLS) +covers the basics of using NFS in a firewalled environment. Additionally, we encourage you to +search for and review the specific documentation for your OS/distro and your firewall software. + +Example for Ubuntu: + +Check that NFS traffic from the client is allowed by the firewall on the host by running +the command: `sudo ufw status`. If it's being blocked, then you can allow traffic from a specific +client with the command below. + +```sh +sudo ufw allow from <client-ip-address> to any port nfs +``` diff --git a/doc/administration/high_availability/pg_ha_architecture.png b/doc/administration/high_availability/pg_ha_architecture.png Binary files differnew file mode 100644 index 00000000000..ef870f652ae --- /dev/null +++ b/doc/administration/high_availability/pg_ha_architecture.png diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md new file mode 100644 index 00000000000..762179cf756 --- /dev/null +++ b/doc/administration/high_availability/pgbouncer.md @@ -0,0 +1,132 @@ +# Working with the bundle Pgbouncer service + +## Overview + +As part of its High Availability stack, GitLab Premium includes a bundled version of [Pgbouncer](https://pgbouncer.github.io/) that can be managed through `/etc/gitlab/gitlab.rb`. + +In a High Availability setup, Pgbouncer is used to seamlessly migrate database connections between servers in a failover scenario. + +Additionally, it can be used in a non-HA setup to pool connections, speeding up response time while reducing resource usage. + +It is recommended to run pgbouncer alongside the `gitlab-rails` service, or on its own dedicated node in a cluster. + +## Operations + +### Running Pgbouncer as part of an HA GitLab installation + +See our [HA documentation for PostgreSQL](database.md) for information on running pgbouncer as part of a HA setup + +### Running Pgbouncer as part of a non-HA GitLab installation + +1. Generate PGBOUNCER_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 pgbouncer` + +1. Generate SQL_USER_PASSWORD_HASH with the command `gitlab-ctl pg-password-md5 gitlab`. We'll also need to enter the plaintext SQL_USER_PASSWORD later + +1. On your database node, ensure the following is set in your `/etc/gitlab/gitlab.rb` + + ```ruby + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' + postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' + postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on + postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node + ``` + +1. Run `gitlab-ctl reconfigure` + + **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. + +1. On the node you are running pgbouncer on, make sure the following is set in `/etc/gitlab/gitlab.rb` + + ```ruby + pgbouncer['enable'] = true + pgbouncer['databases'] = { + gitlabhq_production: { + host: 'DATABASE_HOST', + user: 'pgbouncer', + password: 'PGBOUNCER_USER_PASSWORD_HASH' + } + } + ``` + +1. Run `gitlab-ctl reconfigure` + +1. On the node running unicorn, make sure the following is set in `/etc/gitlab/gitlab.rb` + + ```ruby + gitlab_rails['db_host'] = 'PGBOUNCER_HOST' + gitlab_rails['db_port'] = '6432' + gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' + ``` + +1. Run `gitlab-ctl reconfigure` + +1. At this point, your instance should connect to the database through pgbouncer. If you are having issues, see the [Troubleshooting](#troubleshooting) section + +### Interacting with pgbouncer + +#### Administrative console + +As part of omnibus-gitlab, we provide a command `gitlab-ctl pgb-console` to automatically connect to the pgbouncer administrative console. Please see the [pgbouncer documentation](https://pgbouncer.github.io/usage.html#admin-console) for detailed instructions on how to interact with the console. + +To start a session, run + +```shell +# gitlab-ctl pgb-console +Password for user pgbouncer: +psql (9.6.8, server 1.7.2/bouncer) +Type "help" for help. + +pgbouncer=# +``` + +The password you will be prompted for is the PGBOUNCER_USER_PASSWORD + +To get some basic information about the instance, run +```shell +pgbouncer=# show databases; show clients; show servers; + name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections +---------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- + gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1 + pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 +(2 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link +| remote_pid | tls +------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------ ++------------+----- + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 | +| 0 | + C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 | +| 0 | + C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 | +| 0 | +(8 rows) + + type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem +ote_pid | tls +------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+---- +--------+----- + S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | | + 19980 | +(1 row) +``` + +## Troubleshooting + +In case you are experiencing any issues connecting through pgbouncer, the first place to check is always the logs: + +```shell +# gitlab-ctl tail pgbouncer +``` + +Additionally, you can check the output from `show databases` in the [Administrative console](#administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1. diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index 3daebc4d84b..46ad3ecd9bb 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -1,6 +1,103 @@ -# Configuring Redis for GitLab HA +# Configuring Redis for Scaling and High Availability -> Experimental Redis Sentinel support was [Introduced][ce-1877] in GitLab 8.11. +## Provide your own Redis instance **[CORE ONLY]** + +The following are the requirements for providing your own Redis instance: + +- Redis version 2.8 or higher. Version 3.2 or higher is recommend as this is + what ships with the GitLab Omnibus package. +- Standalone Redis or Redis high availability with Sentinel are supported. Redis + Cluster is not supported. +- Managed Redis from cloud providers such as AWS Elasticache will work. If these + services support high availability, be sure it is not the Redis Cluster type. + +Note the Redis node's IP address or hostname, port, and password (if required). +These will be necessary when configuring the GitLab application servers later. + +## Redis in a Scaled Environment + +This section is relevant for [Scaled Architecture](./README.md#scalable-architecture-examples) +environments including [Basic Scaling](./README.md#basic-scaling) and +[Full Scaling](./README.md#full-scaling). + +### Provide your own Redis instance **[CORE ONLY]** + +If you want to use your own deployed Redis instance(s), +see [Provide your own Redis instance](#provide-your-own-redis-instance-core-only) +for more details. However, you can use the GitLab Omnibus package to easily +deploy the bundled Redis. + +### Standalone Redis using GitLab Omnibus **[CORE ONLY]** + +The GitLab Omnibus package can be used to configure a standalone Redis server. +In this configuration Redis is not highly available, and represents a single +point of failure. However, in a scaled environment the objective is to allow +the environment to handle more users or to increase throughput. Redis itself +is generally stable and can handle many requests so it is an acceptable +trade off to have only a single instance. See [Scaling and High Availability](./README.md) +for an overview of GitLab scaling and high availability options. + +The steps below are the minimum necessary to configure a Redis server with +Omnibus: + +1. SSH into the Redis server. +1. [Download/install](https://about.gitlab.com/installation) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page. + - Do not complete any other steps on the download page. + +1. Edit `/etc/gitlab/gitlab.rb` and add the contents: + + ```ruby + ## Enable Redis + redis['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + unicorn['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + gitlab_monitor['enable'] = false + gitaly['enable'] = false + + redis['bind'] = '0.0.0.0' + redis['port'] = '6379' + redis['password'] = 'SECRET_PASSWORD_HERE' + + gitlab_rails['auto_migrate'] = false + ``` + +1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. +1. Note the Redis node's IP address or hostname, port, and + Redis password. These will be necessary when configuring the GitLab + application servers later. + +Advanced configuration options are supported and can be added if +needed. + +Continue configuration of other components by going +[back to Scaled Architectures](./README.md#scalable-architecture-examples) + +## Redis with High Availability + +This section is relevant for [High Availability Architecture](./README.md#high-availability-architecture-examples) +environments including [Horizontal](./README.md#horizontal), +[Hybrid](./README.md#hybrid), and +[Fully Distributed](./README.md#fully-distributed). + +### Provide your own Redis instance **[CORE ONLY]** + +If you want to use your own deployed Redis instance(s), +see [Provide your own Redis instance](#provide-your-own-redis-instance-core-only) +for more details. However, you can use the GitLab Omnibus package to easily +deploy the bundled Redis. + +### High Availability with GitLab Omnibus **[PREMIUM ONLY]** + +> Experimental Redis Sentinel support was [introduced in GitLab 8.11][ce-1877]. Starting with 8.14, Redis Sentinel is no longer experimental. If you've used it with versions `< 8.14` before, please check the updated documentation here. @@ -53,8 +150,6 @@ failure. Make sure that you read this document once as a whole before configuring the components below. -### High Availability with Sentinel - > **Notes:** > > - Starting with GitLab `8.11`, you can configure a list of Redis Sentinel @@ -270,10 +365,9 @@ The prerequisites for a HA Redis setup are the following: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Enable the master role and disable all other services in the machine - # (you can still enable Sentinel). - redis_master_role['enable'] = true - + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] + # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. # If you really need to bind to an external accessible IP, make @@ -287,6 +381,7 @@ The prerequisites for a HA Redis setup are the following: # Set up password authentication for Redis (use the same password in all nodes). redis['password'] = 'redis-password-goes-here' ``` + 1. Only the primary GitLab application server should handle migrations. To prevent database migrations from running on upgrade, add the following @@ -298,6 +393,10 @@ The prerequisites for a HA Redis setup are the following: 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. +> Note: You can specify multiple roles like sentinel and redis as: +> roles ['redis_sentinel_role', 'redis_master_role']. Read more about high +> availability roles at https://docs.gitlab.com/omnibus/roles/ + ### Step 2. Configuring the slave Redis instances 1. SSH into the **slave** Redis server. @@ -310,11 +409,9 @@ The prerequisites for a HA Redis setup are the following: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Enable the slave role and disable all other services in the machine - # (you can still enable Sentinel). This will also set automatically - # `redis['master'] = false`. - redis_slave_role['enable'] = true - + # Specify server role as 'redis_slave_role' + roles ['redis_slave_role'] + # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. # If you really need to bind to an external accessible IP, make @@ -336,17 +433,19 @@ The prerequisites for a HA Redis setup are the following: #redis['master_port'] = 6379 ``` -1. To prevent database migrations from running on upgrade, run: +1. To prevent reconfigure from running automatically on upgrade, run: ``` sudo touch /etc/gitlab/skip-auto-reconfigure ``` - Only the primary GitLab application server should handle migrations. - 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Go through the steps again for all the other slave nodes. +> Note: You can specify multiple roles like sentinel and redis as: +> roles ['redis_sentinel_role', 'redis_slave_role']. Read more about high +> availability roles at https://docs.gitlab.com/omnibus/roles/ + --- These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -400,13 +499,13 @@ multiple machines with the Sentinel daemon. be duplicate below): ```ruby - redis_sentinel_role['enable'] = true + roles ['redis_sentinel_role'] # Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis' # The same password for Redis authentication you set up for the master node. - redis['password'] = 'redis-password-goes-here' + redis['master_password'] = 'redis-password-goes-here' # The IP of the master Redis node. redis['master_ip'] = '10.0.0.1' @@ -573,8 +672,7 @@ or a failover promotes a different **Master** node. In `/etc/gitlab/gitlab.rb`: ```ruby -redis_master_role['enable'] = true -redis_sentinel_role['enable'] = true +roles ['redis_sentinel_role', 'redis_master_role'] redis['bind'] = '10.0.0.1' redis['port'] = 6379 redis['password'] = 'redis-password-goes-here' @@ -596,8 +694,7 @@ sentinel['quorum'] = 2 In `/etc/gitlab/gitlab.rb`: ```ruby -redis_slave_role['enable'] = true -redis_sentinel_role['enable'] = true +roles ['redis_sentinel_role', 'redis_slave_role'] redis['bind'] = '10.0.0.2' redis['port'] = 6379 redis['password'] = 'redis-password-goes-here' @@ -619,8 +716,7 @@ sentinel['quorum'] = 2 In `/etc/gitlab/gitlab.rb`: ```ruby -redis_slave_role['enable'] = true -redis_sentinel_role['enable'] = true +roles ['redis_sentinel_role', 'redis_slave_role'] redis['bind'] = '10.0.0.3' redis['port'] = 6379 redis['password'] = 'redis-password-goes-here' @@ -643,7 +739,7 @@ In `/etc/gitlab/gitlab.rb`: ```ruby redis['master_name'] = 'gitlab-redis' -redis['password'] = 'redis-password-goes-here' +redis['master_password'] = 'redis-password-goes-here' gitlab_rails['redis_sentinels'] = [ {'host' => '10.0.0.1', 'port' => 26379}, {'host' => '10.0.0.2', 'port' => 26379}, @@ -764,15 +860,11 @@ Before proceeding with the troubleshooting below, check your firewall rules: ### Troubleshooting Redis replication You can check if everything is correct by connecting to each server using -`redis-cli` application, and sending the `INFO` command. +`redis-cli` application, and sending the `info replication` command as below. -If authentication was correctly defined, it should fail with: -`NOAUTH Authentication required` error. Try to authenticate with the -previous defined password with `AUTH redis-password-goes-here` and -try the `INFO` command again. - -Look for the `# Replication` section where you should see some important -information like the `role` of the server. +``` +/opt/gitlab/embedded/bin/redis-cli -a <redis-password> info replication +``` When connected to a `master` redis, you will see the number of connected `slaves`, and a list of each with connection details: @@ -842,7 +934,7 @@ To make sure your configuration is correct: 1. Run in the console: ```ruby - redis = Redis.new(Gitlab::Redis.params) + redis = Redis.new(Gitlab::Redis::SharedState.params) redis.info ``` diff --git a/doc/administration/index.md b/doc/administration/index.md index 986f8146831..02e88dbd2a6 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -72,7 +72,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Branded login page](../customization/branded_login_page.md): Customize the login page with your own logo, title, and description. - [Welcome message](../customization/welcome_message.md): Add a custom welcome message to the sign-in page. - ["New Project" page](../customization/new_project_page.md): Customize the text to be displayed on the page that opens whenever your users create a new project. -- [Additional custom email text](https://docs.gitlab.com/ee/user/admin_area/settings/email.html#custom-additional-text): Add additional custom text to emails sent from GitLab. **[PREMIUM ONLY]** +- [Additional custom email text](https://docs.gitlab.com/ee/user/admin_area/settings/email.html#custom-additional-text-premium-only): Add additional custom text to emails sent from GitLab. **[PREMIUM ONLY]** ### Maintaining GitLab diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 6f23610345f..286b99aceb5 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -93,11 +93,11 @@ This tells the additional processes how often to check for enqueued jobs. ## Starting extra processes via command line Starting extra Sidekiq processes can be done using the command -`/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster`. This command +`/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster`. This command takes arguments using the following syntax: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster [QUEUE,QUEUE,...] [QUEUE, ...] +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster [QUEUE,QUEUE,...] [QUEUE, ...] ``` Each separate argument denotes a group of queues that have to be processed by a @@ -115,14 +115,14 @@ For example, say you want to start 2 extra processes: one to process the done as follows: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit post_receive +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster process_commit post_receive ``` If you instead want to start one process processing both queues you'd use the following syntax: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster process_commit,post_receive ``` If you want to have one Sidekiq process process the "process_commit" and @@ -130,7 +130,7 @@ If you want to have one Sidekiq process process the "process_commit" and you'd use the following: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive gitlab_shell +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster process_commit,post_receive gitlab_shell ``` ### Monitoring @@ -162,7 +162,7 @@ file is written, but this can be changed by passing the `--pidfile` option to `sidekiq-cluster`. For example: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit ``` Keep in mind that the PID file will contain the PID of the `sidekiq-cluster` @@ -199,7 +199,7 @@ one thread per queue up to a maximum of 50. If you wish to change the cap, use the `-m N` option. For example, this would cap the maximum number of threads to 1: ```bash -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive -m 1 +/opt/gitlab/embedded/service/gitlab-rails/ee/bin/sidekiq-cluster process_commit,post_receive -m 1 ``` For each queue group, the concurrency factor will be set to min(number of diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index 36e7c82cc3a..83a7094faaa 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -34,6 +34,7 @@ For example: metrics: script: - echo 'metric_name metric_value' > metrics.txt - reports: - metrics: metrics.txt + artifacts: + reports: + metrics: metrics.txt ``` diff --git a/doc/development/README.md b/doc/development/README.md index 9547f9a285f..83a1145c020 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -104,6 +104,10 @@ description: 'Learn how to contribute to GitLab.' - [Query Count Limits](query_count_limits.md) - [Database helper modules](database_helpers.md) +## Integration guides + +- [Jira Connect app](integrations/jira_connect.md) + ## Testing guides - [Testing standards and style guidelines](testing_guide/index.md) diff --git a/doc/development/geo.md b/doc/development/geo.md index d8669d377b0..c8e6a86eb52 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -2,10 +2,48 @@ Geo connects GitLab instances together. One GitLab instance is designated as a **primary** node and can be run with multiple -**secondary** nodes. Geo orchestrates quite a few components that are -described in more detail below. +**secondary** nodes. Geo orchestrates quite a few components that can be seen on +the diagram below and are described in more detail within this document. -## Database replication + + +## Replication layer + +Geo handles replication for different components: +- [Database](#database-replication): includes the entire application, except cache and jobs. +- [Git repositories](#repository-replication): includes both projects and wikis. +- [Uploaded blobs](#uploads-replication): includes anything from images attached on issues +to raw logs and assets from CI. + +With the exception of the Database replication, on a *secondary* node, everything is coordinated +by the [Geo Log Cursor](#geo-log-cursor). + +### Geo Log Cursor daemon + +The [Geo Log Cursor daemon](#geo-log-cursor-daemon) is a separate process running on +each **secondary** node. It monitors the [Geo Event Log](#geo-event-log) +for new events and creates background jobs for each specific event type. + +For example when a repository is updated, the Geo **primary** node creates +a Geo event with an associated repository updated event. The Geo Log Cursor daemon +picks the event up and schedules a `Geo::ProjectSyncWorker` job which will +use the `Geo::RepositorySyncService` and `Geo::WikiSyncService` classes +to update the repository and the wiki respectively. + +The Geo Log Cursor daemon can operate in High Availability mode automatically. +The daemon will try to acquire a lock from time to time and once acquired, it +will behave as the *active* daemon. + +Any additional running daemons on the same node, will be in standby +mode, ready to resume work if the *active* daemon releases its lock. + +We use the [`ExclusiveLease`](https://www.rubydoc.info/github/gitlabhq/gitlabhq/Gitlab/ExclusiveLease) lock type with a small TTL, that is renewed at every +pooling cycle. That allows us to implement this global lock with a timeout. + +At the end of the pooling cycle, if the daemon can't renew and/or reacquire +the lock, it switches to standby mode. + +### Database replication Geo uses [streaming replication](#streaming-replication) to replicate the database from the **primary** to the **secondary** nodes. This @@ -13,7 +51,7 @@ replication gives the **secondary** nodes access to all the data saved in the database. So users can log in on the **secondary** and read all the issues, merge requests, etc. on the **secondary** node. -## Repository replication +### Repository replication Geo also replicates repositories. Each **secondary** node keeps track of the state of every repository in the [tracking database](#tracking-database). @@ -23,7 +61,7 @@ There are a few ways a repository gets replicated by the: - [Repository Sync worker](#repository-sync-worker). - [Geo Log Cursor](#geo-log-cursor). -### Project Registry +#### Project Registry The `Geo::ProjectRegistry` class defines the model used to track the state of repository replication. For each project in the main @@ -32,15 +70,15 @@ database, one record in the tracking database is kept. It records the following about repositories: - The last time they were synced. -- The last time they were synced successfully. +- The last time they were successfully synced. - If they need to be resynced. -- When retry should be attempted. +- When a retry should be attempted. - The number of retries. -- If and when the they were verified. +- If and when they were verified. It also stores these attributes for project wikis in dedicated columns. -### Repository Sync worker +#### Repository Sync worker The `Geo::RepositorySyncWorker` class runs periodically in the background and it searches the `Geo::ProjectRegistry` model for @@ -59,26 +97,12 @@ times, Geo does a so-called _redownload_. It will do a clean clone into the `@geo-temporary` directory in the root of the storage. When it's successful, we replace the main repo with the newly cloned one. -### Geo Log Cursor - -The [Geo Log Cursor](#geo-log-cursor) is a separate process running on -each **secondary** node. It monitors the [Geo Event Log](#geo-event-log) -and handles all of the events. When it sees an unhandled event, it -starts a background worker to handle that event, depending on the type -of event. - -When a repository receives an update, the Geo **primary** node creates -a Geo event with an associated repository updated event. The cursor -picks that up, and schedules a `Geo::ProjectSyncWorker` job which will -use the `Geo::RepositorySyncService` class and `Geo::WikiSyncService` -class to update the repository and the wiki. - -## Uploads replication +### Uploads replication File uploads are also being replicated to the **secondary** node. To track the state of syncing, the `Geo::FileRegistry` model is used. -### File Registry +#### File Registry Similar to the [Project Registry](#project-registry), there is a `Geo::FileRegistry` model that tracks the synced uploads. @@ -86,7 +110,7 @@ Similar to the [Project Registry](#project-registry), there is a CI Job Artifacts are synced in a similar way as uploads or LFS objects, but they are tracked by `Geo::JobArtifactRegistry` model. -### File Download Dispatch worker +#### File Download Dispatch worker Also similar to the [Repository Sync worker](#repository-sync-worker), there is a `Geo::FileDownloadDispatchWorker` class that is run @@ -113,7 +137,7 @@ Authorization: GL-Geo <access_key>:<JWT payload> ``` The **primary** node uses the `access_key` field to look up the -corresponding Geo **secondary** node and decrypts the JWT payload, +corresponding **secondary** node and decrypts the JWT payload, which contains additional information to identify the file request. This ensures that the **secondary** node downloads the right file for the right database ID. For example, for an LFS object, the @@ -133,6 +157,28 @@ NOTE: **Note:** JWT requires synchronized clocks between the machines involved, otherwise it may fail with an encryption error. +## Git Push to Geo secondary + +The Git Push Proxy exists as a functionality built inside the `gitlab-shell` component. +It is active on a **secondary** node only. It allows the user that has cloned a repository +from the secondary node to push to the same URL. + +Git `push` requests directed to a **secondary** node will be sent over to the **primary** node, +while `pull` requests will continue to be served by the **secondary** node for maximum efficiency. + +HTTPS and SSH requests are handled differently: + +- With HTTPS, we will give the user a `HTTP 302 Redirect` pointing to the project on the **primary** node. +The git client is wise enough to understand that status code and process the redirection. +- With SSH, because there is no equivalent way to perform a redirect, we have to proxy the request. +This is done inside [`gitlab-shell`](https://gitlab.com/gitlab-org/gitlab-shell), by first translating the request +to the HTTP protocol, and then proxying it to the **primary** node. + +The [`gitlab-shell`](https://gitlab.com/gitlab-org/gitlab-shell) daemon knows when to proxy based on the response +from `/api/v4/allowed`. A special `HTTP 300` status code is returned and we execute a "custom action", +specified in the response body. The response contains additional data that allows the proxied `push` operation +to happen on the **primary** node. + ## Using the Tracking Database Along with the main database that is replicated, a Geo **secondary** diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md new file mode 100644 index 00000000000..5bf43d320c6 --- /dev/null +++ b/doc/development/integrations/jira_connect.md @@ -0,0 +1,41 @@ +# Setting up a development environment + +The following are required to install and test the app: + +1. A Jira Cloud instance + + Atlassian provides free instances for development and testing. [Click here to sign up](http://go.atlassian.com/cloud-dev). + +1. A GitLab instance available over the internet + + For the app to work, Jira Cloud should be able to connect to the GitLab instance through the internet. + + To easily expose your local development environment, you can use tools like [serveo](https://serveo.net) or [ngrok](https://ngrok.com). + These also take care of SSL for you because Jira requires all connections to the app host to be over SSL. + +> This feature is currently behind the `:jira_connect_app` feature flag + +# Installing the app in Jira + +1. Enable Jira development mode to install apps that are not from the Atlassian Marketplace + + 1. Navigate to **Jira settings** (cog icon) > **Apps** > **Manage apps**. + 1. Scroll to the bottom of the **Manage apps** page and click **Settings**. + 1. Select **Enable development mode** and click **Apply**. + +1. Install the app + + 1. Navigate to Jira, then choose **Jira settings** (cog icon) > **Apps** > **Manage apps**. + 1. Click **Upload app**. + 1. In the **From this URL** field, provide a link to the app descriptor. The host and port must point to your GitLab instance. + + For example: + ``` + https://xxxx.serveo.net/-/jira_connect/app_descriptor.json + ``` + 1. Click **Upload**. + + If the install was successful, you should see the **GitLab for Jira** app under **Manage apps**. + You can also click **Getting Started** to open the configuration page rendered from your GitLab instance. + + _Note that any changes to the app descriptor requires you to uninstall then reinstall the app._ diff --git a/doc/gitlab-geo/README.md b/doc/gitlab-geo/README.md index d08b47999a9..30d21db7de5 100644 --- a/doc/gitlab-geo/README.md +++ b/doc/gitlab-geo/README.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/index.md' +redirect_to: '../administration/geo/replication/index.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/index.md). +This document was moved to [another location](../administration/geo/replication/index.md). diff --git a/doc/gitlab-geo/after_setup.md b/doc/gitlab-geo/after_setup.md index 90fac0c71d8..c8a7b9d1096 100644 --- a/doc/gitlab-geo/after_setup.md +++ b/doc/gitlab-geo/after_setup.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/using_a_geo_server.md' +redirect_to: '../administration/geo/replication/using_a_geo_server.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/using_a_geo_server.md). +This document was moved to [another location](../administration/geo/replication/using_a_geo_server.md). diff --git a/doc/gitlab-geo/bring-primary-back.md b/doc/gitlab-geo/bring-primary-back.md index eb223367e56..8c43f4d805f 100644 --- a/doc/gitlab-geo/bring-primary-back.md +++ b/doc/gitlab-geo/bring-primary-back.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/disaster_recovery/bring_primary_back.md' +redirect_to: '../administration/geo/disaster_recovery/bring_primary_back.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/disaster_recovery/bring_primary_back.md). +This document was moved to [another location](../administration/geo/disaster_recovery/bring_primary_back.md). diff --git a/doc/gitlab-geo/configuration.md b/doc/gitlab-geo/configuration.md index d0ca191ebeb..b46a2caea4a 100644 --- a/doc/gitlab-geo/configuration.md +++ b/doc/gitlab-geo/configuration.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/configuration.md' +redirect_to: '../administration/geo/replication/configuration.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/configuration.md). +This document was moved to [another location](../administration/geo/replication/configuration.md). diff --git a/doc/gitlab-geo/configuration_source.md b/doc/gitlab-geo/configuration_source.md index e2d96d43ee3..f1aab86fadc 100644 --- a/doc/gitlab-geo/configuration_source.md +++ b/doc/gitlab-geo/configuration_source.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/configuration_source.md' +redirect_to: '../administration/geo/replication/configuration_source.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/configuration_source.md). +This document was moved to [another location](../administration/geo/replication/configuration_source.md). diff --git a/doc/gitlab-geo/database.md b/doc/gitlab-geo/database.md index fe2d830d1d7..b4156dc4ec6 100644 --- a/doc/gitlab-geo/database.md +++ b/doc/gitlab-geo/database.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/database.md' +redirect_to: '../administration/geo/replication/database.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/database.md). +This document was moved to [another location](../administration/geo/replication/database.md). diff --git a/doc/gitlab-geo/database_source.md b/doc/gitlab-geo/database_source.md index 949c4020a32..3392d0f02c0 100644 --- a/doc/gitlab-geo/database_source.md +++ b/doc/gitlab-geo/database_source.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/database_source.md' +redirect_to: '../administration/geo/replication/database_source.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/database_source.md). +This document was moved to [another location](../administration/geo/replication/database_source.md). diff --git a/doc/gitlab-geo/disaster-recovery.md b/doc/gitlab-geo/disaster-recovery.md index 6849df3a4fc..d42e815a879 100644 --- a/doc/gitlab-geo/disaster-recovery.md +++ b/doc/gitlab-geo/disaster-recovery.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/disaster_recovery/index.md' +redirect_to: '../administration/geo/disaster_recovery/index.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/disaster_recovery/index.md). +This document was moved to [another location](../administration/geo/disaster_recovery/index.md). diff --git a/doc/gitlab-geo/docker_registry.md b/doc/gitlab-geo/docker_registry.md index cef4e26beaf..26a708f6845 100644 --- a/doc/gitlab-geo/docker_registry.md +++ b/doc/gitlab-geo/docker_registry.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/docker_registry.md' +redirect_to: '../administration/geo/replication/docker_registry.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/docker_registry.md). +This document was moved to [another location](../administration/geo/replication/docker_registry.md). diff --git a/doc/gitlab-geo/faq.md b/doc/gitlab-geo/faq.md index 1fa26a375d4..f1952bc7e4c 100644 --- a/doc/gitlab-geo/faq.md +++ b/doc/gitlab-geo/faq.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/faq.md' +redirect_to: '../administration/geo/replication/faq.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/faq.md). +This document was moved to [another location](../administration/geo/replication/faq.md). diff --git a/doc/gitlab-geo/ha.md b/doc/gitlab-geo/ha.md index 805b519b8df..23ed11eaf09 100644 --- a/doc/gitlab-geo/ha.md +++ b/doc/gitlab-geo/ha.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/high_availability.md' +redirect_to: '../administration/geo/replication/high_availability.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/high_availability.md). +This document was moved to [another location](../administration/geo/replication/high_availability.md). diff --git a/doc/gitlab-geo/object_storage.md b/doc/gitlab-geo/object_storage.md index 7238d7d0422..1f29b7b7e8c 100644 --- a/doc/gitlab-geo/object_storage.md +++ b/doc/gitlab-geo/object_storage.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/object_storage.md' +redirect_to: '../administration/geo/replication/object_storage.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/object_storage.md). +This document was moved to [another location](../administration/geo/replication/object_storage.md). diff --git a/doc/gitlab-geo/planned-failover.md b/doc/gitlab-geo/planned-failover.md index 9ed42140033..720b6bc9424 100644 --- a/doc/gitlab-geo/planned-failover.md +++ b/doc/gitlab-geo/planned-failover.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/disaster_recovery/planned_failover.md' +redirect_to: '../administration/geo/disaster_recovery/planned_failover.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/disaster_recovery/planned_failover.md). +This document was moved to [another location](../administration/geo/disaster_recovery/planned_failover.md). diff --git a/doc/gitlab-geo/security-review.md b/doc/gitlab-geo/security-review.md index ef0905bb1d3..a0a5b0e536c 100644 --- a/doc/gitlab-geo/security-review.md +++ b/doc/gitlab-geo/security-review.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/security_review.md' +redirect_to: '../administration/geo/replication/security_review.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/security_review.md). +This document was moved to [another location](../administration/geo/replication/security_review.md). diff --git a/doc/gitlab-geo/ssh.md b/doc/gitlab-geo/ssh.md index 310084033ec..4f8db687850 100644 --- a/doc/gitlab-geo/ssh.md +++ b/doc/gitlab-geo/ssh.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.md' +redirect_to: '../administration/operations/fast_ssh_key_lookup.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.md). +This document was moved to [another location](../administration/operations/fast_ssh_key_lookup.md). diff --git a/doc/gitlab-geo/troubleshooting.md b/doc/gitlab-geo/troubleshooting.md index a6bd8b6f650..25fe1372c69 100644 --- a/doc/gitlab-geo/troubleshooting.md +++ b/doc/gitlab-geo/troubleshooting.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/troubleshooting.md' +redirect_to: '../administration/geo/replication/troubleshooting.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/troubleshooting.md). +This document was moved to [another location](../administration/geo/replication/troubleshooting.md). diff --git a/doc/gitlab-geo/tuning.md b/doc/gitlab-geo/tuning.md index b4c44e14776..84ac40f99db 100644 --- a/doc/gitlab-geo/tuning.md +++ b/doc/gitlab-geo/tuning.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/tuning.md' +redirect_to: '../administration/geo/replication/tuning.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/tuning.md). +This document was moved to [another location](../administration/geo/replication/tuning.md). diff --git a/doc/gitlab-geo/updating_the_geo_nodes.md b/doc/gitlab-geo/updating_the_geo_nodes.md index c791668082a..28234ec02ed 100644 --- a/doc/gitlab-geo/updating_the_geo_nodes.md +++ b/doc/gitlab-geo/updating_the_geo_nodes.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/updating_the_geo_nodes.md' +redirect_to: '../administration/geo/replication/updating_the_geo_nodes.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/updating_the_geo_nodes.md). +This document was moved to [another location](../administration/geo/replication/updating_the_geo_nodes.md). diff --git a/doc/gitlab-geo/using_a_geo_server.md b/doc/gitlab-geo/using_a_geo_server.md index 90fac0c71d8..c8a7b9d1096 100644 --- a/doc/gitlab-geo/using_a_geo_server.md +++ b/doc/gitlab-geo/using_a_geo_server.md @@ -1,5 +1,5 @@ --- -redirect_to: 'https://docs.gitlab.com/ee/administration/geo/replication/using_a_geo_server.md' +redirect_to: '../administration/geo/replication/using_a_geo_server.md' --- -This document was moved to [another location](https://docs.gitlab.com/ee/administration/geo/replication/using_a_geo_server.md). +This document was moved to [another location](../administration/geo/replication/using_a_geo_server.md). diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index ca6c92f11f1..7cef664bc98 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -30,7 +30,7 @@ is out of the scope of this document. Once the data is added to the database or repository and [Elasticsearch is enabled in the admin area](#enabling-elasticsearch) the search index will be updated automatically. Elasticsearch can be installed on the same machine as -GitLab, or on a separate server, or you can use the [Amazon Elasticsearch](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html) +GitLab or on a separate server, or you can use the [Amazon Elasticsearch](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html) service. You can follow the steps as described in the [official web site](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html "Elasticsearch installation documentation") or @@ -38,7 +38,7 @@ use the packages that are available for your OS. ## Elasticsearch repository indexer (beta) -In order to improve elasticsearch indexing performance GitLab has made available a [new indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). +In order to improve elasticsearch indexing performance, GitLab has made available a [new indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). This will replace the included Ruby indexer in the future but should be considered beta software for now, so there may be some bugs. If you would like to use it, please follow the instructions below. @@ -56,12 +56,20 @@ installed before running `make`. ##### Debian / Ubuntu -To install on Debian or Ubutu, run: +To install on Debian or Ubuntu, run: ```sh sudo apt install libicu-dev ``` +##### CentOS / RHEL + +To install on CentOS or RHEL, run: + +```sh +sudo yum install libicu-devel +``` + ##### Mac OSX To install on macOS, run: @@ -76,6 +84,8 @@ export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" To build and install the indexer, run: ```sh +git clone https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer.git +cd /gitlab-elasticsearch-indexer make sudo make install ``` @@ -97,10 +107,13 @@ Once installed, enable it under your instance's elasticsearch settings explained Elasticsearch requires additional resources in excess of those documented in the [GitLab system requirements](../install/requirements.md). These will vary by -installation size, but you should ensure **at least** an additional 8 GiB of RAM +installation size, but you should ensure **at least** an additional **8 GiB of RAM** for each Elasticsearch node, per the [official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html). -Storage requirements also vary based on installation side, but as a rule of +Keep in mind, this is the **minimum requirements** as per Elasticsearch. For +production instances, they recommend considerably more resources. + +Storage requirements also vary based on the installation side, but as a rule of thumb, you should allocate the total size of your production database, **plus** two-thirds of the total size of your git repositories. Efforts to reduce this total are being tracked in this epic: [gitlab-org&153](https://gitlab.com/groups/gitlab-org/-/epics/153). @@ -114,7 +127,7 @@ The following Elasticsearch settings are available: | Parameter | Description | | --------- | ----------- | -| `Elasticsearch indexing` | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also keep in mind that this option doesn't have any impact on existing data, this only enables/disables background indexer which tracks data changes. So by enabling this you will not get your existing data indexed, use special rake task for that as explained in [Adding GitLab's data to the Elasticsearch index](#adding-gitlabs-data-to-the-elasticsearch-index). | +| `Elasticsearch indexing` | Enables/disables Elasticsearch indexing. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables background indexer which tracks data changes. So by enabling this you will not get your existing data indexed, use special rake task for that as explained in [Adding GitLab's data to the Elasticsearch index](#adding-gitlabs-data-to-the-elasticsearch-index). | | `Use the new repository indexer (beta)` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). | | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. | | `URL` | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). | @@ -152,22 +165,13 @@ To disable the Elasticsearch integration: 1. Find the 'Elasticsearch' section and uncheck 'Search with Elasticsearch enabled' and 'Elasticsearch indexing' 1. Click **Save** for the changes to take effect +1. [Optional] Delete the existing index by running the command `sudo gitlab-rake gitlab:elastic:delete_index` ## Adding GitLab's data to the Elasticsearch index ### Indexing small instances (database size less than 500 MiB, size of repos less than 5 GiB) -Configure Elasticsearch's host and port in **Admin > Settings > Integrations**. Then create empty indexes using one of the following commands: - -```sh -# Omnibus installations -sudo gitlab-rake gitlab:elastic:create_empty_index - -# Installations from source -bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production -``` - -Then enable Elasticsearch indexing and run repository indexing tasks: +Configure Elasticsearch's host and port in **Admin > Settings**. Then index the data using one of the following commands: ```sh # Omnibus installations @@ -177,12 +181,17 @@ sudo gitlab-rake gitlab:elastic:index bundle exec rake gitlab:elastic:index RAILS_ENV=production ``` -Enable Elasticsearch search. +After it completes the indexing process, [enable Elasticsearch searching](elasticsearch.md#enabling-elasticsearch). ### Indexing large instances +WARNING: **Warning**: +Performing asynchronous indexing, as this will describe, will generate a lot of sidekiq jobs. +Make sure to prepare for this task by either [Horizontally Scaling](../administration/high_availability/README.md#basic-scaling) +or creating [extra sidekiq processes](../administration/operations/extra_sidekiq_processes.md) + NOTE: **Note**: -After indexing the repositories asynchronously, you MUST index the database to be able to search. +After indexing the repositories asynchronously, you **MUST** index the database to be able to search. Configure Elasticsearch's host and port in **Admin > Settings > Integrations**. Then create empty indexes using one of the following commands: @@ -195,8 +204,8 @@ bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production ``` Indexing large Git repositories can take a while. To speed up the process, you -can temporarily disable auto-refreshing and replicating. In our experience you can expect a 20% -time drop. We'll enable them when indexing is done. This step is optional! +can temporarily disable auto-refreshing and replicating. In our experience, you can expect a 20% +decrease in indexing time. We'll enable them when indexing is done. This step is optional! ```bash curl --request PUT localhost:9200/gitlab-production/_settings --data '{ @@ -235,7 +244,7 @@ projects, you may wish to increase the batch size, by setting the `BATCH` environment variable. You can also run the initial indexing synchronously - this is most useful if -you have a small number of projects, or need finer-grained control over indexing +you have a small number of projects or need finer-grained control over indexing than Sidekiq permits: ```sh @@ -293,7 +302,7 @@ bundle exec rake gitlab:elastic:index_repositories UPDATE_INDEX=true ID_TO=1000 ``` You can also use the `gitlab:elastic:clear_index_status` Rake task to force the -indexer to "forget" all progresss, so retrying the indexing process from the +indexer to "forget" all progress, so retrying the indexing process from the start. To index all wikis: @@ -309,7 +318,7 @@ bundle exec rake gitlab:elastic:index_wikis RAILS_ENV=production The wiki indexer also supports the `ID_FROM` and `ID_TO` parameters if you want to limit a project set. -Index all database entities (Keep in mind it can take a while so consider using `screen` or `tmux`): +Index all database entities (Keep in mind it can take a while, so consider using `screen` or `tmux`): ```sh # Omnibus installations @@ -337,6 +346,116 @@ curl --request POST 'http://localhost:9200/_forcemerge?max_num_segments=5' Enable Elasticsearch search in **Admin > Settings > Integrations**. That's it. Enjoy it! +## GitLab Elasticsearch Rake Tasks + +There are several rake tasks available to you via the command line: + +- [sudo gitlab-rake gitlab:elastic:index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This is a wrapper task. It does the following: + - `sudo gitlab-rake gitlab:elastic:create_empty_index` + - `sudo gitlab-rake gitlab:elastic:clear_index_status` + - `sudo gitlab-rake gitlab:elastic:index_wikis` + - `sudo gitlab-rake gitlab:elastic:index_database` + - `sudo gitlab-rake gitlab:elastic:index_repositories` +- [sudo gitlab-rake gitlab:elastic:index_repositories_async](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This iterates over all projects and places them in batches. It then sends these batches to the background via sidekiq jobs to be indexed. +- [sudo gitlab-rake gitlab:elastic:index_repositories_status](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. +- [sudo gitlab-rake gitlab:elastic:index_repositories](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This iterates over all projects and places them in batches. It then performs indexing on said batches synchronously. +- [sudo gitlab-rake gitlab:elastic:index_wikis](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Iterates over every project, determines if said project contains wiki data, and then indexes the blobs (content) of said wiki data. +- [sudo gitlab-rake gitlab:elastic:index_database](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This is a [rake multitask](https://www.rubydoc.info/github/ruby/rake/Rake/MultiTask). It does the following: + - `sudo gitlab-rake gitlab:elastic:index_projects` + - `sudo gitlab-rake gitlab:elastic:index_issues` + - `sudo gitlab-rake gitlab:elastic:index_merge_requests` + - `sudo gitlab-rake gitlab:elastic:index_snippets` + - `sudo gitlab-rake gitlab:elastic:index_notes` + - `sudo gitlab-rake gitlab:elastic:index_milestones` +- [sudo gitlab-rake gitlab:elastic:create_empty_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This generates an empty index on the Elasticsearch side. +- [sudo gitlab-rake gitlab:elastic:clear_index_status](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This deletes all instances of IndexStatus for all projects. +- [sudo gitlab-rake gitlab:elastic:delete_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - This removes the GitLab index on the Elasticsearch instance. +- [sudo gitlab-rake gitlab:elastic:recreate_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Does the same thing as `sudo gitlab-rake gitlab:elastic:create_empty_index` +- [sudo gitlab-rake gitlab:elastic:add_feature_visibility_levels_to_project](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Adds visibility information to the indices for projects. +- [sudo gitlab-rake gitlab:elastic:index_projects](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes projects data. +- [sudo gitlab-rake gitlab:elastic:index_issues](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes issues data. +- [sudo gitlab-rake gitlab:elastic:index_merge_requests](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes merge requests data. +- [sudo gitlab-rake gitlab:elastic:index_snippets](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes the snippets data. +- [sudo gitlab-rake gitlab:elastic:index_notes](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes the notes data. +- [sudo gitlab-rake gitlab:elastic:index_milestones](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Performs an Elasticsearch import that indexes the milestones data. + +### Environment Variables + +In addition to the rake tasks, there are some environment variables that can be used to modify the process: + +| Environment Variable | Data Type | What it does | +| -------------------- |:---------:| ---------------------------------------------------------------------------- | +| `BATCH` | Integer | Modifies the size of the indexing batch (default 300 projects). | +| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). | +| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | +| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | + +### Batching + +The ability to apply batching makes the indexer run more efficiently. The default +size of a batch is 300 projects, which may or may not be ideal for your setup. +Depending on the resources available to your GitLab instance (sidekiq) and your +Elasticsearch instance (RAM, CPU), you may be able to increase or decrease the +batch size for more efficiency. + +- The larger the batch size is, the less sidekiq jobs and indexing requests get created. +- The larger the batch size is, the more time and RAM it takes to process. +- The smaller the batch size, the more sidekiq jobs, and indexing requests get created. +- The smaller the batch size, the more CPU gets utilized. + +Finding the ideal size can be tricky, and will vary from GitLab instance to GitLab instance. +Generally speaking, if the default is not ideal for you, try reducing it to somewhere in +the 50-150 range (for bigger sized repos) or 450-600 range (for many small-sized repos). + +Example use: + +```sh +sudo gitlab-rake gitlab:elastic:index_repositories_async BATCH=50 +``` + +### Indexing a specific project + +Because the `ID_TO` and `ID_FROM` environment variables use the `or equal to` comparison, you can index only one project by using both these variables with the same project ID number: + +```sh +root@git:~# sudo gitlab-rake gitlab:elastic:index_repositories ID_TO=5 ID_FROM=5 +Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... +I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! +``` + +## Elasticsearch Index Scopes + +When performing a search, the GitLab index will use the following scopes: + +| Scope Name | What it searches | +| ---------------- | ---------------------- | +| `commits` | Commit data | +| `projects` | Project data (default) | +| `blobs` | Code | +| `issues` | Issue data | +| `merge_requests` | Merge Request data | +| `milestones` | Milestone data | +| `notes` | Note data | +| `snippets` | Snippet data | +| `wiki_blobs` | Wiki contents | + ## Tuning ### Deleted documents @@ -357,7 +476,7 @@ However, some larger installations may wish to tune the merge policy settings: }' ``` -- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targetd. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. +- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. ```bash curl --request PUT http://localhost:9200/gitlab-production/_settings --data '{ @@ -373,6 +492,18 @@ However, some larger installations may wish to tune the merge policy settings: Here are some common pitfalls and how to overcome them: +- **How can I verify my GitLab instance is using Elasticsearch?** + + The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: + + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term'}) + pp s.search_objects.class.name + ``` + + If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. + - **I updated GitLab and now I can't find anything** We continuously make updates to our indexing strategies and aim to support @@ -382,10 +513,26 @@ Here are some common pitfalls and how to overcome them: - **I indexed all the repositories but I can't find anything** Make sure you indexed all the database data [as stated above](#adding-gitlabs-data-to-the-elasticsearch-index). + + Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. + + If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): + + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term', :scope => ‘blobs’}) + pp s.search_objects.to_a + ``` + + See [Elasticsearch Index Scopes](elasticsearch.md#elasticsearch-index-scopes) for more information on searching for specific types of data. - **I indexed all the repositories but then switched elastic search servers and now I can't find anything** You will need to re-run all the rake tasks to re-index the database, repositories, and wikis. + +- **The indexing process is taking a very long time** + + The more data present in your GitLab instance, the longer the indexing process takes. You might want to try adjusting the BATCH sizes for asynchronous indexing to help speed up the process. - **No new data is added to the Elasticsearch index when I push code** diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index d6d6d9b2517..776ab139c64 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -11,37 +11,38 @@ All Geo nodes have the following settings: | Setting | Description | | --------| ----------- | -| Primary | This marks a Geo Node as primary. There can be only one primary, make sure that you first add the primary node and then all the others. | -| URL | The instance's full URL, in the same way it is configured in `/etc/gitlab/gitlab.rb` (Omnibus GitLab installations) or `gitlab.yml` (source based installations). | +| Primary | This marks a Geo Node as **primary** node. There can be only one **primary** node; make sure that you first add the **primary** node and then all the others. | +| Name | The unique identifier for the Geo node. Must match the setting `gitlab_rails[geo_node_name]` in `/etc/gitlab/gitlab.rb`. The setting defaults to `external_url` with a trailing slash. | +| URL | The instance's user-facing URL. | The node you're reading from is indicated with a green `Current node` label, and -the primary is given a blue `Primary` label. Remember that you can only make -changes on the primary! +the **primary** node is given a blue `Primary` label. Remember that you can only make +changes on the **primary** node! -## Secondary node settings +## **Secondary** node settings -Secondaries have a number of additional settings available: +**Secondary** nodes have a number of additional settings available: | Setting | Description | |---------------------------|-------------| - Selective synchronization | Enable Geo [selective sync](https://docs.gitlab.com/ee/administration/geo/replication/configuration.html#selective-synchronization) for this **secondary** node. | +| Selective synchronization | Enable Geo [selective sync](https://docs.gitlab.com/ee/administration/geo/replication/configuration.html#selective-synchronization) for this **secondary** node. | | Repository sync capacity | Number of concurrent requests this **secondary** node will make to the **primary** node when backfilling repositories. | | File sync capacity | Number of concurrent requests this **secondary** node will make to the **primary** node when backfilling files. | ## Geo backfill -Secondaries are notified of changes to repositories and files by the primary, +**Secondary** nodes are notified of changes to repositories and files by the **primary** node, and will always attempt to synchronize those changes as quickly as possible. -Backfill is the act of populating the secondary with repositories and files that -existed *before* the secondary was added to the database. Since there may be +Backfill is the act of populating the **secondary** node with repositories and files that +existed *before* the **secondary** node was added to the database. Since there may be extremely large numbers of repositories and files, it's infeasible to attempt to download them all at once, so GitLab places an upper limit on the concurrency of these operations. How long the backfill takes is a function of the maximum concurrency, but higher -values place more strain on the primary node. From [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3107), -the limits are configurable - if your primary node has lots of surplus capacity, +values place more strain on the **primary** node. From [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3107), +the limits are configurable. If your **primary** node has lots of surplus capacity, you can increase the values to complete backfill in a shorter time. If it's under heavy load and backfill is reducing its availability for normal requests, you can decrease them. @@ -55,3 +56,15 @@ which is used by users. Internal URL does not need to be a private address. Internal URL defaults to External URL, but you can customize it under **Admin area > Geo Nodes**. + +## Multiple secondary nodes behind a load balancer + +In GitLab 11.11, **secondary** nodes can use identical external URLs as long as +a unique `name` is set for each Geo node. The `gitlab.rb` setting +`gitlab_rails[geo_node_name]` must: + +- Be set for each GitLab instance that runs `unicorn`, `sidekiq`, or `geo_logcursor`. +- Match a Geo node name. + +The load balancer must use sticky sessions in order to avoid authentication +failures and cross site request errors. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 29db6fc8958..2d0c2be4233 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -201,6 +201,7 @@ dependency_scanning: --env DS_ANALYZER_IMAGE_PREFIX \ --env DS_ANALYZER_IMAGE_TAG \ --env DS_DEFAULT_ANALYZERS \ + --env DS_EXCLUDED_PATHS \ --env DEP_SCAN_DISABLE_REMOTE_CHECKS \ --env DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ --env DS_PULL_ANALYZER_IMAGE_TIMEOUT \ diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 377d218321a..02c115b7f22 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -171,6 +171,8 @@ sast: --env SAST_ANALYZER_IMAGE_PREFIX \ --env SAST_ANALYZER_IMAGE_TAG \ --env SAST_DEFAULT_ANALYZERS \ + --env SAST_EXCLUDED_PATHS \ + --env SAST_BANDIT_EXCLUDED_PATHS \ --env SAST_BRAKEMAN_LEVEL \ --env SAST_GOSEC_LEVEL \ --env SAST_FLAWFINDER_LEVEL \ diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index bfc3e3a7de0..9003018a521 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -24,7 +24,7 @@ in the label’s title, using the format `key::value`. For example:  -Two scoped labels with the same key but a different value cannot simultaneeously +Two scoped labels with the same key but a different value cannot simultaneously apply to an issue, epic, or merge request. For example, if an issue already has `priority::3` and you apply `priority::2` to it, `priority::3` is automatically removed from the issue. diff --git a/lib/api/api.rb b/lib/api/api.rb index a572cca24e9..f4a96b9711b 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,7 +6,7 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") - NO_SLASH_URL_PART_REGEX = %r{[^/]+} + NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index 6d01d51955c..cce79e73d2d 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -2,13 +2,13 @@ module Banzai module ColorParser - ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 - PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% - ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ - BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 - DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] - RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad - HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/.freeze # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/.freeze # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/.freeze + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/.freeze # 00..255 + DEGS = /-?\d+(?:deg)?/i.freeze # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i.freeze # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/.freeze RGB_FORMAT = %r{ (?:rgba? \( @@ -20,7 +20,7 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze HSL_FORMAT = %r{ (?:hsla? \( @@ -28,11 +28,11 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze - COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix.freeze # Public: Analyzes whether the String is a color code. # diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 086adf59d2b..56214043d87 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -33,7 +33,7 @@ module Banzai # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # # Rubular: http://rubular.com/r/nrL3r9yUiq - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)}.freeze # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index a27d18facd1..544231adea4 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -20,7 +20,7 @@ module Banzai \s* ^\k<delim> # closing front matter marker \s* - }mx + }mx.freeze def call html.sub(PATTERN) do |_match| diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 50bf823929c..ee7f10ebdf6 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -33,7 +33,7 @@ module Banzai (?<new_link>.+?) (?<title>\ ".+?")? \) - }x + }x.freeze # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index f2ae17b44fa..1a68d773048 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -17,7 +17,7 @@ module Banzai # :toc - String containing Table of Contents data as a `ul` element with # `li` child elements. class TableOfContentsFilter < HTML::Pipeline::Filter - PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u + PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze def call return doc if context[:no_header_anchors] diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 239780d8626..9b7d1548056 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module DeclarativePolicy diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 1204e53ee2e..d301efc3205 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -36,8 +36,8 @@ module Gitlab end COM_URL = 'https://gitlab.com'.freeze - APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} - SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} + APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze + SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 78fa25c5516..4ebf2afb9cb 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a84f794bfae..1924f2ffee2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -8,8 +8,8 @@ module Gitlab self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze # These regex patterns are tested against a relative path, relative to # the upload directory. diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 81ca2b0a9b7..1ee44a3a5a9 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -16,7 +16,7 @@ module Gitlab RELATIVE_UPLOAD_DIR ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 7011dd1aaf2..1c3ce08be76 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -11,8 +11,8 @@ module Gitlab ParserError = Class.new(StandardError) InvalidStreamError = Class.new(StandardError) - VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ - INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/.freeze + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}.freeze attr_reader :stream, :path, :full_version diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 03af99ba9a5..c911bfa7ff6 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 7d6e0704d4a..df92e229f12 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -7,7 +7,7 @@ module Gitlab class Skip < Chain::Base include ::Gitlab::Utils::StrongMemoize - SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i.freeze def perform! if skipped? diff --git a/lib/gitlab/content_disposition.rb b/lib/gitlab/content_disposition.rb index 32207514ce5..ff6154a5b26 100644 --- a/lib/gitlab/content_disposition.rb +++ b/lib/gitlab/content_disposition.rb @@ -22,13 +22,13 @@ module Gitlab end # rubocop:disable Style/VariableInterpolation - TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/.freeze def ascii_filename 'filename="' + percent_escape(::I18n.transliterate(filename), TRADITIONAL_ESCAPED_CHAR) + '"' end - RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/.freeze # rubocop:enable Style/VariableInterpolation def utf8_filename diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 68890aa8e30..3ef19d801b7 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -103,8 +103,6 @@ module Gitlab none: "", qa: "~QA" }.freeze - - # rubocop:disable Style/RegexpLiteral CATEGORIES = { %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :none, # To reinstate roulette for documentation, set to `:docs`. @@ -151,7 +149,6 @@ module Gitlab %r{\.(md|txt)\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\.js\z} => :frontend }.freeze - # rubocop:enable Style/RegexpLiteral end end end diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index b46d4ba0b02..ef9b1bf5224 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -14,7 +14,7 @@ module Gitlab 'GL-HOOK-ERR:' # Messages marked as safe by user ].freeze - SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/ + SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze def initialize(message = '') super(sanitize(message)) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index d34b50c5215..4908f236cd1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,7 +26,7 @@ module Gitlab end end - PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index be1334ca98a..d336b1ba797 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 070e3b2db8d..5b98ce7d5ed 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index 62a5216d159..184083de2bc 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index d32a6980eb8..4586b1d94a7 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 9e4d70a71ff..30f181542be 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric - JITTER_RANGE = 0.000001..0.001 + JITTER_RANGE = (0.000001..0.001).freeze attr_reader :series, :values, :tags, :type diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 83c52a6c6e0..8e17073abab 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -3,7 +3,7 @@ module Gitlab module Middleware class ReadOnly - API_VERSIONS = (3..4) + API_VERSIONS = (3..4).freeze def self.internal_routes @internal_routes ||= diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb index 849cf8f759b..0719fb2e8c6 100644 --- a/lib/gitlab/middleware/release_env.rb +++ b/lib/gitlab/middleware/release_env.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 810aba436cc..3137676ba4b 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -15,7 +15,7 @@ module Gitlab mr: :merge_request }).freeze - OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/ + OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze attr_reader :options diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index f5176376a60..4a62e83e8e9 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -11,7 +11,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}.freeze def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index 0d4e6be2129..98f78c5e74b 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -9,7 +9,7 @@ module Gitlab class Scrubber < Loofah::Scrubber # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes - DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u + DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u.freeze def scrub(node) unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 747cb0f9142..f7b08d58e49 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -4,9 +4,9 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = %r{text/html|application/json}i + CONTENT_TYPES = %r{text/html|application/json}i.freeze - IGNORE_PATHS = %r{^/sherlock} + IGNORE_PATHS = %r{^/sherlock}.freeze def initialize(app) @app = app diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 11561eec32a..159ce27e702 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -15,7 +15,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix.freeze # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 607c9c8dec1..a66a2e0726b 100644 --- a/lib/gitlab/slash_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index b698391c8bd..fd108b4c124 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do def fuzzy_search(query, columns) diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index f0557f6ad68..ede60c9ab1d 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -7,7 +7,7 @@ module Gitlab class UserExtractor # Not using `Devise.email_regexp` to filter out any chars that an email # does not end with and not pinning the email to a start of end of a string. - EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/.freeze USERNAME_REGEXP = User.reference_pattern def initialize(text) diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index 2e98227a05e..1b17162f71d 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true unless Rails.env.production? diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 560a52053d8..8cf7c9e89f0 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -209,7 +209,7 @@ namespace :gemojione do image.destroy! end - EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i + EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i.freeze def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) # Rename file from unicode to emoji name matches = EMOJI_IMAGE_PATH_RE.match(image_path) diff --git a/qa/qa/page/project/settings/ci_cd.rb b/qa/qa/page/project/settings/ci_cd.rb index 44a62cf6ccf..b8c5c563da6 100644 --- a/qa/qa/page/project/settings/ci_cd.rb +++ b/qa/qa/page/project/settings/ci_cd.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module QA diff --git a/qa/qa/runtime/key/ecdsa.rb b/qa/qa/runtime/key/ecdsa.rb index 6e5c0a2deca..46a1e5f54cf 100644 --- a/qa/qa/runtime/key/ecdsa.rb +++ b/qa/qa/runtime/key/ecdsa.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module QA diff --git a/qa/qa/runtime/key/ed25519.rb b/qa/qa/runtime/key/ed25519.rb index 53b556d744d..3a3567d55da 100644 --- a/qa/qa/runtime/key/ed25519.rb +++ b/qa/qa/runtime/key/ed25519.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module QA diff --git a/rubocop/cop/gitlab/finder_with_find_by.rb b/rubocop/cop/gitlab/finder_with_find_by.rb index f45a37ddc06..764a5073143 100644 --- a/rubocop/cop/gitlab/finder_with_find_by.rb +++ b/rubocop/cop/gitlab/finder_with_find_by.rb @@ -2,7 +2,7 @@ module RuboCop module Cop module Gitlab class FinderWithFindBy < RuboCop::Cop::Cop - FIND_PATTERN = /\Afind(_by\!?)?\z/ + FIND_PATTERN = /\Afind(_by\!?)?\z/.freeze ALLOWED_MODULES = ['FinderMethods'].freeze def message(used_method) @@ -36,7 +36,7 @@ module RuboCop def find_on_execute?(node) chained_on_node = node.descendants.first node.method_name.to_s =~ FIND_PATTERN && - chained_on_node&.method_name == :execute + chained_on_node.is_a?(RuboCop::AST::SendNode) && chained_on_node.method_name == :execute end def allowed_module?(node) diff --git a/rubocop/rubocop.rb b/rubocop/rubocop.rb index ce6bdbf292c..e2a19978839 100644 --- a/rubocop/rubocop.rb +++ b/rubocop/rubocop.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName require_relative 'cop/gitlab/module_with_instance_variables' require_relative 'cop/gitlab/predicate_memoization' require_relative 'cop/gitlab/httparty' diff --git a/spec/initializers/secret_token_spec.rb b/spec/initializers/secret_token_spec.rb index 6366be30079..726ce07a2d1 100644 --- a/spec/initializers/secret_token_spec.rb +++ b/spec/initializers/secret_token_spec.rb @@ -6,8 +6,8 @@ describe 'create_tokens' do let(:secrets) { ActiveSupport::OrderedOptions.new } - HEX_KEY = /\h{128}/ - RSA_KEY = /\A-----BEGIN RSA PRIVATE KEY-----\n.+\n-----END RSA PRIVATE KEY-----\n\Z/m + HEX_KEY = /\h{128}/.freeze + RSA_KEY = /\A-----BEGIN RSA PRIVATE KEY-----\n.+\n-----END RSA PRIVATE KEY-----\n\Z/m.freeze before do allow(File).to receive(:write) diff --git a/spec/lib/gitlab/path_regex_spec.rb b/spec/lib/gitlab/path_regex_spec.rb index 857862a2abd..84b2e2dc823 100644 --- a/spec/lib/gitlab/path_regex_spec.rb +++ b/spec/lib/gitlab/path_regex_spec.rb @@ -120,10 +120,10 @@ describe Gitlab::PathRegex do # - Followed by one or more path-parts not starting with `:` or `*` # - Followed by a path-part that includes a wildcard parameter `*` # At the time of writing these routes match: http://rubular.com/r/Rv2pDE5Dvw - STARTING_WITH_NAMESPACE = %r{^/\*namespace_id/:(project_)?id} - NON_PARAM_PARTS = %r{[^:*][a-z\-_/]*} - ANY_OTHER_PATH_PART = %r{[a-z\-_/:]*} - WILDCARD_SEGMENT = /\*/ + STARTING_WITH_NAMESPACE = %r{^/\*namespace_id/:(project_)?id}.freeze + NON_PARAM_PARTS = %r{[^:*][a-z\-_/]*}.freeze + ANY_OTHER_PATH_PART = %r{[a-z\-_/:]*}.freeze + WILDCARD_SEGMENT = /\*/.freeze let(:namespaced_wildcard_routes) do routes_without_format.select do |p| p =~ %r{#{STARTING_WITH_NAMESPACE}/#{NON_PARAM_PARTS}/#{ANY_OTHER_PATH_PART}#{WILDCARD_SEGMENT}} @@ -144,7 +144,7 @@ describe Gitlab::PathRegex do end.uniq end - STARTING_WITH_GROUP = %r{^/groups/\*(group_)?id/} + STARTING_WITH_GROUP = %r{^/groups/\*(group_)?id/}.freeze let(:group_routes) do routes_without_format.select do |path| path =~ STARTING_WITH_GROUP |
