From a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 16 Jun 2021 18:25:58 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-0-stable-ee --- doc/.vale/gitlab/Acronyms.yml | 19 +- doc/.vale/gitlab/SubstitutionWarning.yml | 12 + doc/.vale/gitlab/Substitutions.yml | 2 +- doc/.vale/gitlab/spelling-exceptions.txt | 7 +- doc/administration/audit_events.md | 53 +- doc/administration/audit_reports.md | 4 +- doc/administration/auth/ldap/index.md | 212 +- .../auth/ldap/ldap-troubleshooting.md | 89 +- doc/administration/auth/okta.md | 1 + doc/administration/clusters/kas.md | 2 +- doc/administration/compliance.md | 36 +- doc/administration/configure.md | 16 + doc/administration/consul.md | 21 +- doc/administration/database_load_balancing.md | 55 +- doc/administration/environment_variables.md | 4 +- doc/administration/external_pipeline_validation.md | 19 +- doc/administration/feature_flags.md | 5 +- doc/administration/file_hooks.md | 11 +- doc/administration/geo/disaster_recovery/index.md | 27 +- .../geo/disaster_recovery/planned_failover.md | 24 +- doc/administration/geo/index.md | 25 +- doc/administration/geo/replication/datatypes.md | 60 +- doc/administration/geo/replication/faq.md | 10 +- .../geo/replication/geo_validation_tests.md | 4 +- .../geo/replication/multiple_servers.md | 48 +- .../geo/replication/remove_geo_node.md | 1 + .../geo/replication/security_review.md | 2 +- .../geo/replication/troubleshooting.md | 87 +- .../geo/replication/using_a_geo_server.md | 1 + .../geo/replication/version_specific_updates.md | 10 + doc/administration/geo/setup/database.md | 232 +- doc/administration/geo/setup/external_database.md | 41 +- doc/administration/git_annex.md | 1 + doc/administration/gitaly/configure_gitaly.md | 3 +- doc/administration/gitaly/faq.md | 90 + doc/administration/gitaly/index.md | 184 +- doc/administration/gitaly/praefect.md | 160 +- doc/administration/incoming_email.md | 13 +- doc/administration/index.md | 5 +- doc/administration/instance_limits.md | 83 +- doc/administration/integration/kroki.md | 25 +- doc/administration/integration/plantuml.md | 2 +- doc/administration/integration/terminal.md | 13 +- doc/administration/issue_closing_pattern.md | 2 +- doc/administration/job_artifacts.md | 4 +- doc/administration/job_logs.md | 12 +- doc/administration/libravatar.md | 4 +- doc/administration/load_balancer.md | 2 +- doc/administration/logs.md | 81 +- doc/administration/maintenance_mode/index.md | 30 +- .../gitlab_self_monitoring_project/index.md | 13 +- .../monitoring/performance/gitlab_configuration.md | 3 +- .../performance/grafana_configuration.md | 7 +- doc/administration/monitoring/performance/index.md | 1 - .../monitoring/performance/performance_bar.md | 23 +- .../monitoring/performance/request_profiling.md | 2 +- .../monitoring/prometheus/gitlab_metrics.md | 31 +- doc/administration/monitoring/prometheus/index.md | 34 +- doc/administration/nfs.md | 11 +- doc/administration/object_storage.md | 186 +- .../operations/cleaning_up_redis_sessions.md | 2 +- .../operations/extra_sidekiq_processes.md | 127 +- .../operations/extra_sidekiq_routing.md | 164 ++ .../operations/fast_ssh_key_lookup.md | 17 +- .../operations/filesystem_benchmarking.md | 2 +- doc/administration/operations/index.md | 4 +- doc/administration/operations/puma.md | 7 +- .../operations/sidekiq_memory_killer.md | 16 +- doc/administration/operations/ssh_certificates.md | 2 +- doc/administration/operations/unicorn.md | 116 +- doc/administration/packages/container_registry.md | 105 +- doc/administration/packages/dependency_proxy.md | 17 + doc/administration/pages/index.md | 118 +- doc/administration/polling.md | 2 +- doc/administration/postgresql/external.md | 4 + .../postgresql/img/pg_ha_architecture.png | Bin 18412 -> 18308 bytes doc/administration/postgresql/index.md | 2 +- doc/administration/postgresql/pgbouncer.md | 2 +- .../postgresql/replication_and_failover.md | 535 +---- doc/administration/postgresql/standalone.md | 15 +- doc/administration/pseudonymizer.md | 2 +- doc/administration/raketasks/check.md | 42 +- doc/administration/read_only_gitlab.md | 10 +- .../redis/replication_and_failover.md | 62 +- .../redis/replication_and_failover_external.md | 22 +- .../reference_architectures/10k_users.md | 483 ++-- .../reference_architectures/25k_users.md | 430 ++-- .../reference_architectures/2k_users.md | 136 +- .../reference_architectures/3k_users.md | 366 +-- .../reference_architectures/50k_users.md | 423 ++-- .../reference_architectures/5k_users.md | 356 +-- .../reference_architectures/index.md | 4 +- .../reference_architectures/troubleshooting.md | 207 +- doc/administration/reply_by_email_postfix_setup.md | 2 +- doc/administration/repository_checks.md | 9 +- doc/administration/repository_storage_paths.md | 13 +- doc/administration/repository_storage_types.md | 4 +- doc/administration/sidekiq.md | 5 +- doc/administration/smime_signing_email.md | 2 +- .../static_objects_external_storage.md | 3 +- doc/administration/terraform_state.md | 2 +- doc/administration/timezone.md | 2 +- doc/administration/troubleshooting/debug.md | 29 +- doc/administration/troubleshooting/defcon.md | 20 +- .../troubleshooting/diagnostics_tools.md | 2 +- .../troubleshooting/gitlab_rails_cheat_sheet.md | 134 +- .../troubleshooting/group_saml_scim.md | 4 + .../img/azure_configure_group_claim.png | Bin 0 -> 45870 bytes doc/administration/troubleshooting/index.md | 2 +- .../troubleshooting/kubernetes_cheat_sheet.md | 4 +- .../troubleshooting/linux_cheat_sheet.md | 6 +- doc/administration/troubleshooting/log_parsing.md | 2 +- .../navigating_gitlab_via_rails_console.md | 4 +- doc/administration/troubleshooting/postgresql.md | 3 +- doc/administration/troubleshooting/sidekiq.md | 2 +- doc/administration/troubleshooting/ssl.md | 2 +- .../troubleshooting/test_environments.md | 2 +- .../troubleshooting/tracing_correlation_id.md | 2 +- doc/administration/whats-new.md | 3 +- doc/api/README.md | 11 +- doc/api/applications.md | 4 +- doc/api/audit_events.md | 2 + doc/api/boards.md | 3 +- doc/api/broadcast_messages.md | 11 +- doc/api/commits.md | 18 +- doc/api/container_registry.md | 30 +- doc/api/custom_attributes.md | 3 +- doc/api/dependency_proxy.md | 3 +- doc/api/deploy_keys.md | 13 +- doc/api/deploy_tokens.md | 14 +- doc/api/deployments.md | 100 +- doc/api/discussions.md | 24 +- doc/api/dora/metrics.md | 35 +- doc/api/dora4_group_analytics.md | 1 + doc/api/environments.md | 6 +- doc/api/feature_flag_specs.md | 292 +-- doc/api/feature_flags_legacy.md | 314 +-- doc/api/graphql/getting_started.md | 127 +- doc/api/graphql/index.md | 7 +- doc/api/graphql/reference/index.md | 1026 ++++++--- doc/api/graphql/removed_items.md | 26 + doc/api/group_badges.md | 7 +- doc/api/group_clusters.md | 2 +- doc/api/group_import_export.md | 7 +- doc/api/group_labels.md | 7 +- doc/api/group_level_variables.md | 11 +- doc/api/group_protected_environments.md | 154 ++ doc/api/group_relations_export.md | 6 +- doc/api/group_repository_storage_moves.md | 12 +- doc/api/group_wikis.md | 9 +- doc/api/groups.md | 124 +- doc/api/instance_level_ci_variables.md | 8 +- doc/api/invitations.md | 9 +- doc/api/issues.md | 5 +- doc/api/job_artifacts.md | 16 +- doc/api/jobs.md | 44 +- doc/api/labels.md | 35 +- doc/api/lint.md | 19 +- doc/api/managed_licenses.md | 3 +- doc/api/members.md | 7 +- doc/api/merge_request_approvals.md | 156 +- doc/api/merge_requests.md | 23 + doc/api/merge_trains.md | 2 +- doc/api/notification_settings.md | 4 +- doc/api/oauth2.md | 3 +- doc/api/packages/composer.md | 11 +- doc/api/packages/nuget.md | 2 +- doc/api/packages/pypi.md | 77 +- doc/api/pages_domains.md | 23 +- doc/api/pipeline_schedules.md | 40 +- doc/api/pipeline_triggers.md | 10 +- doc/api/pipelines.md | 24 +- doc/api/project_aliases.md | 10 +- doc/api/project_badges.md | 4 +- doc/api/project_clusters.md | 21 +- doc/api/project_import_export.md | 35 +- doc/api/project_level_variables.md | 8 +- doc/api/project_repository_storage_moves.md | 6 +- doc/api/project_snippets.md | 8 +- doc/api/projects.md | 248 +- doc/api/protected_branches.md | 28 +- doc/api/protected_environments.md | 5 +- doc/api/protected_tags.md | 2 +- doc/api/releases/index.md | 3 +- doc/api/releases/links.md | 4 +- doc/api/remote_mirrors.md | 3 +- doc/api/repositories.md | 5 + doc/api/repository_files.md | 29 +- doc/api/resource_access_tokens.md | 16 +- doc/api/resource_label_events.md | 7 +- doc/api/runners.md | 40 +- doc/api/scim.md | 18 +- doc/api/search.md | 9 +- doc/api/services.md | 4 +- doc/api/settings.md | 7 +- doc/api/snippet_repository_storage_moves.md | 12 +- doc/api/status_checks.md | 201 ++ doc/api/suggestions.md | 2 + doc/api/system_hooks.md | 6 +- doc/api/tags.md | 85 +- doc/api/templates/dockerfiles.md | 4 - doc/api/templates/gitlab_ci_ymls.md | 2 +- doc/api/todos.md | 3 + doc/api/users.md | 57 +- doc/api/vulnerability_exports.md | 14 +- doc/api/wikis.md | 15 +- .../ci_scale/ci_builds_cumulative_forecast.png | Bin 36221 -> 19274 bytes .../ci_scale/ci_builds_daily_forecast.png | Bin 29472 -> 11550 bytes doc/architecture/blueprints/ci_scale/index.md | 10 +- .../index.md | 614 +++++ .../container_registry_metadata_database/index.md | 6 +- doc/ci/README.md | 4 +- doc/ci/caching/index.md | 114 +- doc/ci/chatops/README.md | 1 + .../bitbucket_integration.md | 2 +- .../ci_cd_for_external_repos/github_integration.md | 2 +- doc/ci/ci_cd_for_external_repos/index.md | 2 +- doc/ci/directed_acyclic_graph/index.md | 2 +- doc/ci/docker/README.md | 1 + doc/ci/docker/index.md | 2 +- doc/ci/docker/using_docker_build.md | 9 +- doc/ci/docker/using_docker_images.md | 4 + doc/ci/docker/using_kaniko.md | 8 +- doc/ci/enable_or_disable_ci.md | 2 +- doc/ci/environments/deployment_safety.md | 4 +- doc/ci/environments/environments_dashboard.md | 4 +- doc/ci/environments/index.md | 32 +- doc/ci/environments/protected_environments.md | 136 +- doc/ci/examples/README.md | 6 +- .../authenticating-with-hashicorp-vault/index.md | 5 + .../laravel_with_gitlab_and_envoy/index.md | 22 +- doc/ci/examples/php.md | 2 +- ...test-and-deploy-python-application-to-heroku.md | 1 + .../test-and-deploy-ruby-application-to-heroku.md | 1 + doc/ci/examples/test-clojure-application.md | 1 + doc/ci/git_submodules.md | 4 +- doc/ci/interactive_web_terminal/index.md | 2 +- doc/ci/introduction/index.md | 2 +- doc/ci/jobs/index.md | 6 +- doc/ci/jobs/job_control.md | 391 +++- doc/ci/large_repositories/index.md | 16 +- doc/ci/merge_request_pipelines/index.md | 18 +- .../pipelines_for_merged_results/index.md | 4 +- .../merge_trains/index.md | 7 +- doc/ci/migration/circleci.md | 14 +- doc/ci/migration/jenkins.md | 20 +- doc/ci/multi_project_pipelines.md | 2 +- doc/ci/parent_child_pipelines.md | 4 +- doc/ci/pipelines/index.md | 13 +- doc/ci/pipelines/job_artifacts.md | 12 +- doc/ci/pipelines/pipeline_architectures.md | 4 +- doc/ci/pipelines/pipeline_artifacts.md | 13 +- doc/ci/pipelines/pipeline_efficiency.md | 4 +- doc/ci/pipelines/schedules.md | 23 +- doc/ci/pipelines/settings.md | 26 +- doc/ci/quick_start/README.md | 1 + doc/ci/quick_start/index.md | 18 +- doc/ci/review_apps/index.md | 2 +- doc/ci/runners/README.md | 1037 ++------- doc/ci/runners/configure_runners.md | 601 +++++ doc/ci/runners/runners_scope.md | 251 ++ doc/ci/services/README.md | 1 + doc/ci/services/gitlab.md | 2 +- doc/ci/ssh_keys/README.md | 1 + doc/ci/ssh_keys/index.md | 2 +- doc/ci/triggers/README.md | 38 +- doc/ci/troubleshooting.md | 8 +- doc/ci/unit_test_reports.md | 23 +- doc/ci/variables/README.md | 258 +-- doc/ci/variables/predefined_variables.md | 6 +- doc/ci/variables/where_variables_can_be_used.md | 11 +- doc/ci/yaml/README.md | 623 ++--- doc/ci/yaml/gitlab_ci_yaml.md | 2 +- doc/ci/yaml/includes.md | 22 +- doc/ci/yaml/script.md | 2 +- doc/development/README.md | 4 +- doc/development/api_graphql_styleguide.md | 41 +- doc/development/api_styleguide.md | 8 +- doc/development/application_limits.md | 2 +- doc/development/appsec/index.md | 2 +- doc/development/architecture.md | 81 +- doc/development/auto_devops.md | 8 +- doc/development/background_migrations.md | 115 +- doc/development/changelog.md | 333 +-- doc/development/chaos_endpoints.md | 19 +- .../cicd/cicd_reference_documentation_guide.md | 2 +- doc/development/cicd/index.md | 22 +- doc/development/cicd/templates.md | 20 + doc/development/code_review.md | 8 +- doc/development/contributing/index.md | 2 +- doc/development/contributing/issue_workflow.md | 10 +- .../contributing/merge_request_workflow.md | 14 +- doc/development/dangerbot.md | 2 +- doc/development/database/pagination_guidelines.md | 12 +- .../database/strings_and_the_text_data_type.md | 8 +- doc/development/db_dump.md | 2 +- doc/development/deprecation_guidelines/index.md | 5 + doc/development/diffs.md | 18 +- .../documentation/graphql_styleguide.md | 2 +- doc/development/documentation/index.md | 28 +- .../documentation/restful_api_styleguide.md | 18 +- .../site_architecture/deployment_process.md | 18 +- .../documentation/site_architecture/global_nav.md | 20 +- .../documentation/site_architecture/index.md | 16 +- .../site_architecture/release_process.md | 1 + doc/development/documentation/structure.md | 2 +- doc/development/documentation/styleguide/index.md | 85 +- .../documentation/styleguide/word_list.md | 163 ++ doc/development/documentation/testing.md | 27 +- doc/development/documentation/workflow.md | 6 +- doc/development/ee_features.md | 6 +- doc/development/elasticsearch.md | 31 +- doc/development/event_tracking/backend.md | 3 +- doc/development/event_tracking/frontend.md | 3 +- doc/development/event_tracking/index.md | 3 +- .../experiment_guide/experimentation.md | 5 +- .../experiment_guide/gitlab_experiment.md | 20 + doc/development/experiment_guide/index.md | 26 +- doc/development/fe_guide/accessibility.md | 50 +- doc/development/fe_guide/content_editor.md | 116 + doc/development/fe_guide/design_anti_patterns.md | 4 +- doc/development/fe_guide/editor_lite.md | 1 + doc/development/fe_guide/event_tracking.md | 3 +- doc/development/fe_guide/frontend_faq.md | 10 +- doc/development/fe_guide/graphql.md | 11 +- .../img/content_editor_highlevel_diagram.png | Bin 0 -> 47794 bytes doc/development/fe_guide/index.md | 5 + doc/development/fe_guide/style/vue.md | 9 +- doc/development/fe_guide/troubleshooting.md | 8 +- doc/development/fe_guide/vue.md | 44 +- doc/development/fe_guide/vuex.md | 3 +- doc/development/feature_flags/controls.md | 33 +- doc/development/feature_flags/development.md | 2 + doc/development/feature_flags/index.md | 12 +- doc/development/feature_flags/process.md | 1 + doc/development/file_storage.md | 2 +- doc/development/geo.md | 8 +- doc/development/go_guide/index.md | 2 +- doc/development/gotchas.md | 2 +- doc/development/graphql_guide/authorization.md | 2 +- doc/development/graphql_guide/pagination.md | 22 +- doc/development/i18n/externalization.md | 330 +-- doc/development/i18n/index.md | 53 +- doc/development/i18n/merging_translations.md | 53 +- doc/development/i18n/proofreader.md | 49 +- doc/development/i18n/translation.md | 104 +- doc/development/import_project.md | 2 +- doc/development/integrations/codesandbox.md | 2 +- doc/development/integrations/jenkins.md | 3 +- doc/development/integrations/secure.md | 2 +- .../integrations/secure_partner_integration.md | 4 +- doc/development/internal_api.md | 36 +- doc/development/jh_features_review.md | 6 + doc/development/kubernetes.md | 25 +- doc/development/licensing.md | 2 +- doc/development/logging.md | 6 +- doc/development/maintenance_mode.md | 4 +- .../merge_request_performance_guidelines.md | 10 +- doc/development/migration_style_guide.md | 42 +- doc/development/module_with_instance_variables.md | 14 +- doc/development/multi_version_compatibility.md | 133 +- doc/development/namespaces_storage_statistics.md | 4 +- doc/development/new_fe_guide/dependencies.md | 1 + .../new_fe_guide/development/performance.md | 2 +- .../new_fe_guide/modules/dirty_submit.md | 2 +- doc/development/packages.md | 6 +- doc/development/performance.md | 4 +- doc/development/permissions.md | 2 +- doc/development/pipelines.md | 49 +- doc/development/polymorphic_associations.md | 14 +- .../product_analytics/event_dictionary.md | 3 +- doc/development/product_analytics/index.md | 3 +- doc/development/profiling.md | 2 +- doc/development/prometheus_metrics.md | 2 +- doc/development/pry_debugging.md | 6 +- doc/development/query_recorder.md | 2 +- doc/development/rake_tasks.md | 6 +- doc/development/reactive_caching.md | 2 +- doc/development/real_time.md | 97 + doc/development/redis.md | 15 +- doc/development/reference_processing.md | 16 +- doc/development/reusing_abstractions.md | 6 +- doc/development/routing.md | 12 + doc/development/scalability.md | 8 +- doc/development/secure_coding_guidelines.md | 4 +- doc/development/sidekiq_style_guide.md | 68 + doc/development/single_table_inheritance.md | 39 + doc/development/snowplow.md | 2 + doc/development/snowplow/index.md | 130 +- doc/development/stage_group_dashboards.md | 59 +- doc/development/testing_guide/best_practices.md | 56 +- doc/development/testing_guide/ci.md | 4 +- .../testing_guide/end_to_end/beginners_guide.md | 6 +- .../testing_guide/end_to_end/feature_flags.md | 4 +- .../end_to_end/img/gl-capybara_V13_12.png | Bin 19201 -> 6777 bytes .../end_to_end/img/gl-chemlab_V13_12.png | Bin 17753 -> 6580 bytes doc/development/testing_guide/end_to_end/index.md | 7 +- .../running_tests_that_require_special_setup.md | 4 +- doc/development/testing_guide/flaky_tests.md | 6 +- doc/development/testing_guide/frontend_testing.md | 22 +- doc/development/testing_guide/review_apps.md | 16 +- doc/development/testing_guide/testing_levels.md | 4 +- .../testing_guide/testing_rake_tasks.md | 8 +- doc/development/understanding_explain_plans.md | 4 +- doc/development/uploads.md | 8 +- doc/development/usage_ping.md | 2 + doc/development/usage_ping/dictionary.md | 2408 +++++++++++--------- doc/development/usage_ping/index.md | 15 +- doc/development/usage_ping/metrics_dictionary.md | 35 +- .../usage_ping/metrics_instrumentation.md | 40 +- doc/development/utilities.md | 30 +- doc/development/what_requires_downtime.md | 1 + doc/development/wikis.md | 2 +- doc/downgrade_ee_to_ce/README.md | 1 + doc/downgrade_ee_to_ce/index.md | 6 +- doc/gitlab-basics/README.md | 1 + doc/gitlab-basics/add-file.md | 2 +- doc/gitlab-basics/create-branch.md | 4 +- doc/gitlab-basics/create-project.md | 1 + doc/gitlab-basics/create-your-ssh-keys.md | 1 + doc/gitlab-basics/fork-project.md | 1 + doc/gitlab-basics/start-using-git.md | 552 +++-- doc/install/README.md | 1 + doc/install/azure/index.md | 3 +- doc/install/google_cloud_platform/index.md | 4 +- doc/install/installation.md | 38 +- doc/install/next_steps.md | 46 +- doc/install/postgresql_extensions.md | 22 +- doc/install/relative_url.md | 4 +- doc/install/requirements.md | 104 +- doc/integration/README.md | 1 + doc/integration/akismet.md | 6 +- doc/integration/elasticsearch.md | 105 +- doc/integration/external-issue-tracker.md | 7 - doc/integration/gitpod.md | 3 +- doc/integration/google_workspace_saml.md | 1 + doc/integration/jira/dvcs.md | 16 +- doc/integration/jira_development_panel.md | 1 + doc/integration/kerberos.md | 32 +- doc/integration/oauth_provider.md | 15 +- doc/integration/omniauth.md | 4 +- doc/integration/saml.md | 4 +- doc/integration/slash_commands.md | 2 +- doc/integration/sourcegraph.md | 3 +- doc/intro/README.md | 1 + doc/legal/README.md | 1 + doc/legal/index.md | 2 +- doc/migrate_ci_to_ce/README.md | 462 +--- doc/operations/error_tracking.md | 8 +- doc/operations/feature_flags.md | 24 +- .../incident_management/alert_integrations.md | 1 + doc/operations/incident_management/alerts.md | 10 +- .../img/pagerduty_incidents_integration_v13_3.png | Bin 43318 -> 13288 bytes doc/operations/incident_management/incidents.md | 20 +- doc/operations/incident_management/integrations.md | 24 +- .../incident_management/oncall_schedules.md | 12 +- doc/operations/incident_management/paging.md | 6 +- doc/operations/incident_management/status_page.md | 11 +- doc/operations/index.md | 6 +- doc/operations/metrics/alerts.md | 32 +- .../metrics_dashboard_template_selection_v13_3.png | Bin 31905 -> 9033 bytes doc/operations/metrics/dashboards/index.md | 48 +- doc/operations/metrics/dashboards/settings.md | 4 +- doc/operations/metrics/embed_grafana.md | 2 +- doc/operations/metrics/index.md | 34 +- doc/operations/product_analytics.md | 2 +- doc/operations/tracing.md | 4 +- doc/policy/maintenance.md | 28 +- doc/push_rules/push_rules.md | 21 +- doc/raketasks/README.md | 1 + doc/raketasks/backup_restore.md | 12 +- doc/security/README.md | 4 +- doc/security/asset_proxy.md | 8 +- doc/security/cicd_environment_variables.md | 1 + doc/security/cicd_variables.md | 14 +- doc/security/crime_vulnerability.md | 4 +- doc/security/information_exclusivity.md | 2 +- doc/security/password_length_limits.md | 19 +- doc/security/password_storage.md | 25 +- ...swords_for_integrated_authentication_methods.md | 2 +- ...ject_import_decompressed_archive_size_limits.md | 6 +- doc/security/rack_attack.md | 2 +- doc/security/rate_limits.md | 2 +- doc/security/ssh_keys_restrictions.md | 15 +- doc/security/two_factor_authentication.md | 57 +- doc/security/unlock_user.md | 2 +- doc/security/user_email_confirmation.md | 10 +- doc/security/webhooks.md | 21 +- doc/subscriptions/bronze_starter.md | 6 +- doc/subscriptions/gitlab_com/index.md | 2 +- doc/subscriptions/img/license-file.png | Bin 0 -> 72190 bytes doc/subscriptions/img/license-overview.png | Bin 0 -> 38018 bytes doc/subscriptions/img/publicly-visible.png | Bin 0 -> 58946 bytes doc/subscriptions/img/support-diagram.png | Bin 0 -> 49941 bytes doc/subscriptions/index.md | 102 +- doc/subscriptions/self_managed/index.md | 30 +- doc/system_hooks/system_hooks.md | 2 +- doc/tools/email.md | 9 +- doc/topics/authentication/index.md | 12 +- doc/topics/autodevops/customize.md | 65 +- doc/topics/autodevops/index.md | 10 +- doc/topics/autodevops/quick_start_guide.md | 30 +- doc/topics/autodevops/requirements.md | 77 +- doc/topics/autodevops/stages.md | 130 +- .../upgrading_auto_deploy_dependencies.md | 20 +- doc/topics/build_your_application.md | 16 + doc/topics/git/bisect.md | 6 +- doc/topics/git/cherry_picking.md | 2 +- doc/topics/git/feature_branch_development.md | 12 +- doc/topics/git/feature_branching.md | 6 +- doc/topics/git/getting_started.md | 6 +- doc/topics/git/git_add.md | 6 +- doc/topics/git/git_log.md | 6 +- doc/topics/git/git_rebase.md | 19 +- doc/topics/git/lfs/index.md | 44 +- .../git/lfs/migrate_from_git_annex_to_git_lfs.md | 1 + doc/topics/git/lfs/migrate_to_git_lfs.md | 6 +- doc/topics/git/merge_conflicts.md | 6 +- doc/topics/git/merge_requests.md | 1 + .../img/rebase_reset.png | Bin 21836 -> 33399 bytes .../img/revert.png | Bin 13243 -> 21473 bytes .../numerous_undo_possibilities_in_git/index.md | 218 +- doc/topics/git/rollback_commits.md | 6 +- doc/topics/git/stash.md | 6 +- doc/topics/git/subtree.md | 6 +- doc/topics/git/tags.md | 2 +- doc/topics/git/unstage.md | 6 +- doc/topics/git/useful_git_commands.md | 7 +- doc/topics/gitlab_flow.md | 81 +- doc/topics/plan_and_track.md | 28 + doc/topics/release_your_application.md | 13 + doc/topics/set_up_organization.md | 16 + doc/topics/use_gitlab.md | 19 + doc/university/README.md | 1 + doc/university/index.md | 1 + doc/university/training/gitlab_flow.md | 1 + doc/university/training/index.md | 1 + doc/university/training/topics/agile_git.md | 1 + doc/university/training/topics/bisect.md | 1 + doc/university/training/topics/cherry_picking.md | 1 + doc/university/training/topics/env_setup.md | 1 + .../training/topics/feature_branching.md | 1 + doc/university/training/topics/getting_started.md | 1 + doc/university/training/topics/git_add.md | 1 + doc/university/training/topics/git_intro.md | 1 + doc/university/training/topics/git_log.md | 1 + doc/university/training/topics/merge_conflicts.md | 1 + doc/university/training/topics/merge_requests.md | 1 + doc/university/training/topics/rollback_commits.md | 1 + doc/university/training/topics/stash.md | 1 + doc/university/training/topics/subtree.md | 1 + doc/university/training/topics/tags.md | 1 + doc/university/training/topics/unstage.md | 1 + doc/university/training/user_training.md | 1 + doc/update/README.md | 1 + doc/update/index.md | 20 +- doc/update/mysql_to_postgresql.md | 8 +- doc/update/patch_versions.md | 2 +- doc/update/upgrading_from_ce_to_ee.md | 2 +- doc/update/upgrading_from_source.md | 20 +- doc/update/upgrading_postgresql_using_slony.md | 2 +- doc/user/abuse_reports.md | 1 + doc/user/admin_area/abuse_reports.md | 1 + .../admin_area/activating_deactivating_users.md | 1 + doc/user/admin_area/analytics/dev_ops_report.md | 61 +- .../analytics/img/admin_devops_adoption_v14_0.png | Bin 0 -> 52947 bytes .../instance_activity_pipelines_chart_v13_6_a.png | Bin 92540 -> 30831 bytes doc/user/admin_area/analytics/index.md | 15 +- doc/user/admin_area/analytics/usage_trends.md | 2 +- doc/user/admin_area/analytics/user_cohorts.md | 1 + doc/user/admin_area/appearance.md | 6 +- doc/user/admin_area/approving_users.md | 8 +- doc/user/admin_area/blocking_unblocking_users.md | 1 + doc/user/admin_area/broadcast_messages.md | 79 +- doc/user/admin_area/credentials_inventory.md | 9 +- doc/user/admin_area/custom_project_templates.md | 10 +- doc/user/admin_area/diff_limits.md | 32 +- doc/user/admin_area/geo_nodes.md | 2 +- doc/user/admin_area/img/cohorts_v13_9_a.png | Bin 122096 -> 35297 bytes doc/user/admin_area/index.md | 60 +- doc/user/admin_area/license.md | 14 +- doc/user/admin_area/merge_requests_approvals.md | 4 +- doc/user/admin_area/moderate_users.md | 59 +- doc/user/admin_area/monitoring/health_check.md | 8 +- doc/user/admin_area/review_abuse_reports.md | 8 +- .../settings/account_and_limit_settings.md | 82 +- .../admin_area/settings/continuous_integration.md | 140 +- .../admin_area/settings/external_authorization.md | 7 +- doc/user/admin_area/settings/floc.md | 3 +- doc/user/admin_area/settings/gitaly_timeouts.md | 8 +- doc/user/admin_area/settings/help_page.md | 6 +- .../settings/img/admin_required_pipeline.png | Bin 22587 -> 0 bytes ...ous_integration_shared_runner_details_v14_0.png | Bin 0 -> 80106 bytes .../settings/img/custom_sign_in_page_v13_6.png | Bin 61203 -> 0 bytes doc/user/admin_area/settings/img/enforce_terms.png | Bin 54881 -> 156968 bytes .../settings/import_export_rate_limits.md | 4 +- doc/user/admin_area/settings/index.md | 22 +- .../settings/instance_template_repository.md | 13 +- .../settings/package_registry_rate_limits.md | 5 +- .../settings/project_integration_management.md | 6 +- .../settings/push_event_activities_limit.md | 13 +- .../settings/rate_limit_on_notes_creation.md | 2 +- .../settings/rate_limits_on_raw_endpoints.md | 7 +- .../admin_area/settings/sign_in_restrictions.md | 16 +- .../admin_area/settings/sign_up_restrictions.md | 18 +- doc/user/admin_area/settings/terms.md | 5 +- doc/user/admin_area/settings/third_party_offers.md | 6 +- doc/user/admin_area/settings/usage_statistics.md | 16 +- .../admin_area/settings/user_and_ip_rate_limits.md | 15 +- .../settings/visibility_and_access_controls.md | 12 +- doc/user/admin_area/user_cohorts.md | 6 +- doc/user/analytics/ci_cd_analytics.md | 25 +- doc/user/analytics/index.md | 2 +- doc/user/analytics/value_stream_analytics.md | 132 +- .../api_fuzzing/create_har_files.md | 8 +- doc/user/application_security/api_fuzzing/index.md | 69 +- .../container_scanning/index.md | 330 +-- .../application_security/coverage_fuzzing/index.md | 4 +- doc/user/application_security/cve_id_request.md | 2 +- .../application_security/dast/browser_based.md | 75 +- .../dast/img/dast_auth_browser_scan_highlight.png | Bin 0 -> 203980 bytes .../img/dast_auth_browser_scan_search_elements.png | Bin 0 -> 155046 bytes .../dast/img/dast_auth_report.jpg | Bin 0 -> 121121 bytes doc/user/application_security/dast/index.md | 785 ++++--- doc/user/application_security/dast_api/index.md | 67 +- .../dependency_scanning/analyzers.md | 2 +- .../dependency_scanning/index.md | 12 +- ...urity_approval_rules_and_enabled_jobs_v13_4.png | Bin 35553 -> 0 bytes ...ured_security_approval_rules_and_jobs_v13_4.png | Bin 29773 -> 0 bytes doc/user/application_security/index.md | 165 +- .../offline_deployments/index.md | 19 +- doc/user/application_security/sast/analyzers.md | 23 +- doc/user/application_security/sast/index.md | 36 +- .../application_security/secret_detection/index.md | 16 +- .../img/security_center_settings_v13_4.png | Bin 69604 -> 23188 bytes .../security_dashboard/index.md | 3 +- doc/user/application_security/terminology/index.md | 6 +- .../threat_monitoring_policy_alert_list_v13_12.png | Bin 59862 -> 22929 bytes .../threat_monitoring/index.md | 58 +- ...ty_page_merge_request_button_dropdown_v13_1.png | Bin 53561 -> 0 bytes .../application_security/vulnerabilities/index.md | 146 +- .../vulnerabilities/severities.md | 7 +- .../vulnerability_report/index.md | 4 +- doc/user/clusters/agent/index.md | 4 +- doc/user/clusters/agent/repository.md | 115 +- doc/user/clusters/agent/runner.md | 1 + doc/user/clusters/applications.md | 735 +----- doc/user/clusters/cost_management.md | 8 +- doc/user/clusters/crossplane.md | 2 +- doc/user/clusters/img/fluentd_v13_0.png | Bin 32445 -> 0 bytes doc/user/clusters/integrations.md | 27 +- doc/user/clusters/management_project.md | 8 +- doc/user/clusters/management_project_template.md | 86 + .../migrating_from_gma_to_project_template.md | 95 + doc/user/compliance/license_compliance/index.md | 42 +- doc/user/discussions/index.md | 9 +- doc/user/gitlab_com/index.md | 302 +-- doc/user/group/bulk_editing/index.md | 1 + doc/user/group/clusters/index.md | 14 +- doc/user/group/custom_project_templates.md | 57 +- .../img/group_devops_adoption_v13_11.png | Bin 58389 -> 0 bytes .../img/group_devops_adoption_v14_0.png | Bin 0 -> 52890 bytes doc/user/group/devops_adoption/index.md | 26 +- doc/user/group/epics/epic_boards.md | 170 +- doc/user/group/epics/img/epic_board_v13_10.png | Bin 42037 -> 0 bytes doc/user/group/epics/img/epic_board_v14_0.png | Bin 0 -> 16512 bytes doc/user/group/epics/manage_epics.md | 27 +- doc/user/group/import/index.md | 7 +- doc/user/group/index.md | 68 +- doc/user/group/insights/index.md | 2 +- .../img/issues_created_per_month_v12_8_a.png | Bin 54332 -> 16244 bytes doc/user/group/repositories_analytics/index.md | 4 +- .../saml_sso/img/member_enterprise_badge_v14_0.png | Bin 0 -> 31502 bytes .../group/saml_sso/img/saml_group_links_v13_9.png | Bin 0 -> 162931 bytes doc/user/group/saml_sso/index.md | 21 +- doc/user/group/saml_sso/scim_setup.md | 4 + doc/user/group/settings/import_export.md | 10 +- doc/user/group/subgroups/index.md | 23 +- .../img/delete_value_stream_v13_12.png | Bin 57007 -> 20164 bytes .../img/new_value_stream_v13_12.png | Bin 55481 -> 19993 bytes .../img/vsa_filter_bar_v13_12.png | Bin 125695 -> 36706 bytes .../img/vsa_label_based_stage_v14_0.png | Bin 65149 -> 19873 bytes .../img/vsa_overview_stage_v13_11.png | Bin 60960 -> 21148 bytes .../img/vsa_time_metrics_v13_12.png | Bin 51065 -> 18354 bytes doc/user/group/value_stream_analytics/index.md | 4 +- doc/user/img/completed_tasks_v13_3.png | Bin 10844 -> 14835 bytes doc/user/infrastructure/index.md | 15 +- doc/user/infrastructure/terraform_state.md | 17 +- doc/user/instance/clusters/index.md | 5 +- doc/user/markdown.md | 454 ++-- doc/user/operations_dashboard/index.md | 2 +- doc/user/packages/composer_repository/index.md | 4 +- doc/user/packages/conan_repository/index.md | 2 +- doc/user/packages/container_registry/index.md | 15 +- doc/user/packages/index.md | 11 +- doc/user/packages/infrastructure_registry/index.md | 93 + doc/user/packages/maven_repository/index.md | 15 +- doc/user/packages/npm_registry/index.md | 9 +- doc/user/packages/nuget_repository/index.md | 2 +- doc/user/packages/package_registry/index.md | 39 +- doc/user/packages/pypi_repository/index.md | 39 +- doc/user/packages/rubygems_registry/index.md | 4 +- .../packages/terraform_module_registry/index.md | 124 + doc/user/packages/workflows/project_registry.md | 4 +- .../packages/workflows/working_with_monorepos.md | 64 + doc/user/permissions.md | 104 +- doc/user/profile/account/create_accounts.md | 23 +- doc/user/profile/account/delete_account.md | 5 +- .../profile/account/two_factor_authentication.md | 13 +- .../img/notification_global_settings_v13_12.png | Bin 20731 -> 9268 bytes .../profile/img/unknown_sign_in_email_v13_1.png | Bin 20230 -> 0 bytes .../profile/img/unknown_sign_in_email_v14_0.png | Bin 0 -> 68946 bytes doc/user/profile/index.md | 16 +- doc/user/profile/notifications.md | 1 + doc/user/profile/unknown_sign_in_notification.md | 2 +- doc/user/project/badges.md | 31 +- doc/user/project/bulk_editing.md | 1 + doc/user/project/canary_deployments.md | 4 +- doc/user/project/clusters/add_eks_clusters.md | 13 +- doc/user/project/clusters/add_gke_clusters.md | 12 +- doc/user/project/clusters/add_remove_clusters.md | 62 +- doc/user/project/clusters/index.md | 47 +- doc/user/project/clusters/kubernetes_pod_logs.md | 24 +- .../protect/container_host_security/index.md | 6 +- .../container_host_security/quick_start_guide.md | 24 +- .../protect/container_network_security/index.md | 6 +- .../quick_start_guide.md | 34 +- doc/user/project/clusters/protect/index.md | 5 +- .../guide_waf_ingress_disabled_settings_v12_10.png | Bin 51416 -> 0 bytes .../img/guide_waf_ingress_installation_v12_10.png | Bin 44243 -> 0 bytes .../img/guide_waf_ingress_save_changes_v12_10.png | Bin 54688 -> 0 bytes .../protect/web_application_firewall/index.md | 103 - .../web_application_firewall/quick_start_guide.md | 265 --- doc/user/project/clusters/runbooks/index.md | 99 +- doc/user/project/clusters/serverless/index.md | 74 +- doc/user/project/deploy_boards.md | 2 +- doc/user/project/deploy_keys/index.md | 2 +- doc/user/project/deploy_tokens/index.md | 2 +- doc/user/project/description_templates.md | 39 +- doc/user/project/file_lock.md | 8 +- doc/user/project/highlighting.md | 2 +- ...owners_approval_new_protected_branch_v13_10.png | Bin 30238 -> 0 bytes ...ode_owners_approval_protected_branch_v13_10.png | Bin 17263 -> 0 bytes doc/user/project/import/bitbucket.md | 2 +- doc/user/project/import/bitbucket_server.md | 2 +- doc/user/project/import/clearcase.md | 2 +- doc/user/project/import/cvs.md | 2 +- doc/user/project/import/fogbugz.md | 2 +- doc/user/project/import/gemnasium.md | 1 + doc/user/project/import/gitea.md | 2 +- doc/user/project/import/github.md | 2 +- doc/user/project/import/gitlab_com.md | 2 +- doc/user/project/import/index.md | 2 +- doc/user/project/import/manifest.md | 2 +- doc/user/project/import/perforce.md | 2 +- doc/user/project/import/phabricator.md | 2 +- doc/user/project/import/repo_by_url.md | 2 +- doc/user/project/import/svn.md | 6 +- doc/user/project/import/tfvc.md | 6 +- doc/user/project/index.md | 4 +- .../integrations/gitlab_slack_application.md | 2 +- doc/user/project/integrations/hipchat.md | 7 - doc/user/project/integrations/index.md | 2 +- doc/user/project/integrations/jira.md | 1 + .../integrations/jira_cloud_configuration.md | 1 + doc/user/project/integrations/jira_integrations.md | 1 + .../integrations/jira_server_configuration.md | 1 + .../integrations/mattermost_slash_commands.md | 6 +- doc/user/project/integrations/overview.md | 15 +- doc/user/project/integrations/prometheus.md | 144 +- .../integrations/prometheus_library/kubernetes.md | 2 +- .../prometheus_library/nginx_ingress.md | 22 - .../prometheus_library/nginx_ingress_vts.md | 22 - .../project/integrations/services_templates.md | 68 +- doc/user/project/integrations/webex_teams.md | 3 +- doc/user/project/integrations/webhooks.md | 14 +- doc/user/project/issue_board.md | 199 +- doc/user/project/issues/confidential_issues.md | 8 +- doc/user/project/issues/csv_export.md | 105 +- doc/user/project/issues/csv_import.md | 2 +- doc/user/project/issues/design_management.md | 7 +- doc/user/project/issues/due_dates.md | 24 +- .../issues/img/issue_type_change_v13_12.png | Bin 0 -> 52414 bytes doc/user/project/issues/issue_data_and_actions.md | 2 +- doc/user/project/issues/managing_issues.md | 24 +- doc/user/project/labels.md | 2 +- .../img/access_requests_management_v13_9.png | Bin 24246 -> 0 bytes .../members/img/add_user_email_accept_v13_9.png | Bin 21877 -> 0 bytes .../members/img/add_user_email_ready_v13_8.png | Bin 28850 -> 0 bytes .../members/img/add_user_email_search_v13_8.png | Bin 29293 -> 0 bytes .../members/img/withdraw_access_request_button.png | Bin 28154 -> 0 bytes doc/user/project/members/index.md | 203 +- .../project/members/share_project_with_groups.md | 2 +- .../merge_requests/accessibility_testing.md | 12 +- .../project/merge_requests/allow_collaboration.md | 50 +- doc/user/project/merge_requests/approvals/index.md | 32 +- doc/user/project/merge_requests/approvals/rules.md | 6 +- .../project/merge_requests/approvals/settings.md | 2 +- .../authorization_for_merge_requests.md | 16 +- .../merge_requests/browser_performance_testing.md | 31 +- doc/user/project/merge_requests/changes.md | 60 +- .../project/merge_requests/cherry_pick_changes.md | 25 +- doc/user/project/merge_requests/code_quality.md | 19 +- doc/user/project/merge_requests/commits.md | 28 + .../merge_requests/creating_merge_requests.md | 3 +- doc/user/project/merge_requests/getting_started.md | 58 +- .../merge_requests/img/allow_collaboration.png | Bin 10806 -> 0 bytes .../img/allow_collaboration_after_save.png | Bin 5410 -> 0 bytes .../img/code_quality_mr_diff_report_v14.png | Bin 0 -> 54803 bytes .../merge_requests/img/commit-button_v13_12.png | Bin 0 -> 8834 bytes .../merge_requests/img/conflict_ui_v14_0.png | Bin 0 -> 8371 bytes .../merge_requests/img/merge_request_pipeline.png | Bin 0 -> 31026 bytes .../img/project_merge_requests_list_view_v13_5.png | Bin 0 -> 87738 bytes .../img/reviewer_approval_rules_form_v13_8.png | Bin 42245 -> 0 bytes .../img/reviewer_approval_rules_sidebar_v13_8.png | Bin 38840 -> 0 bytes .../img/status_checks_branches_selector_v14_0.png | Bin 0 -> 5460 bytes .../img/status_checks_create_form_v14_0.png | Bin 0 -> 11913 bytes .../img/status_checks_delete_modal_v14_0.png | Bin 0 -> 5662 bytes .../img/status_checks_list_view_v14_0.png | Bin 0 -> 15958 bytes .../img/status_checks_update_form_v14_0.png | Bin 0 -> 13348 bytes doc/user/project/merge_requests/index.md | 85 +- .../merge_requests/load_performance_testing.md | 10 +- .../merge_requests/merge_request_approvals.md | 1 + .../merge_requests/merge_request_dependencies.md | 2 +- .../merge_requests/merge_when_pipeline_succeeds.md | 2 +- .../project/merge_requests/resolve_conflicts.md | 4 +- .../reviewing_and_managing_merge_requests.md | 1 + .../reviews/img/merge_request_pipeline.png | Bin 31026 -> 0 bytes .../img/project_merge_requests_list_view_v13_5.png | Bin 87738 -> 0 bytes .../img/reviewer_approval_rules_form_v13_8.png | Bin 0 -> 42245 bytes .../img/reviewer_approval_rules_sidebar_v13_8.png | Bin 0 -> 38840 bytes doc/user/project/merge_requests/reviews/index.md | 143 +- .../project/merge_requests/reviews/suggestions.md | 4 + .../project/merge_requests/squash_and_merge.md | 2 +- doc/user/project/merge_requests/status_checks.md | 179 ++ .../merge_requests/test_coverage_visualization.md | 84 +- .../testing_and_reports_in_merge_requests.md | 2 +- doc/user/project/merge_requests/versions.md | 2 +- doc/user/project/merge_requests/widgets.md | 64 + .../work_in_progress_merge_requests.md | 1 + doc/user/project/milestones/index.md | 4 +- doc/user/project/new_ci_build_permissions_model.md | 1 + .../dns_concepts.md | 2 +- .../custom_domains_ssl_tls_certification/index.md | 4 +- .../lets_encrypt_integration.md | 4 +- .../ssl_tls_concepts.md | 2 +- .../pages/getting_started/pages_ci_cd_template.md | 4 +- .../getting_started/pages_forked_sample_project.md | 2 +- .../pages/getting_started/pages_from_scratch.md | 2 +- .../getting_started/pages_new_project_template.md | 3 +- doc/user/project/pages/getting_started_part_one.md | 2 +- doc/user/project/pages/index.md | 2 +- doc/user/project/pages/introduction.md | 4 +- .../project/pages/lets_encrypt_for_gitlab_pages.md | 2 +- doc/user/project/pages/pages_access_control.md | 2 +- doc/user/project/pages/redirects.md | 2 +- doc/user/project/protected_branches.md | 40 +- doc/user/project/protected_tags.md | 2 +- doc/user/project/quick_actions.md | 1 - doc/user/project/releases/index.md | 17 +- doc/user/project/repository/branches/default.md | 3 +- .../repository/img/download_source_code.png | Bin 19681 -> 0 bytes .../repository/img/file_ext_icons_repo_v12_10.png | Bin 73624 -> 0 bytes doc/user/project/repository/index.md | 418 ++-- .../project/repository/jupyter_notebooks/index.md | 9 +- .../project/repository/repository_mirroring.md | 46 +- doc/user/project/repository/web_editor.md | 8 +- doc/user/project/settings/import_export.md | 12 +- doc/user/project/settings/index.md | 94 +- doc/user/project/settings/project_access_tokens.md | 2 + doc/user/project/time_tracking.md | 6 +- doc/user/project/web_ide/index.md | 26 +- doc/user/project/wiki/img/content_editor_v14.0.png | Bin 0 -> 13771 bytes .../wiki/img/use_new_editor_button_v14.0.png | Bin 0 -> 16719 bytes doc/user/project/wiki/index.md | 82 +- doc/user/project/working_with_projects.md | 25 +- doc/user/reserved_names.md | 6 +- doc/user/search/advanced_search.md | 42 +- doc/user/search/index.md | 4 +- doc/user/shortcuts.md | 8 +- doc/user/snippets.md | 8 +- doc/user/todos.md | 1 + 882 files changed, 17324 insertions(+), 15393 deletions(-) create mode 100644 doc/administration/configure.md create mode 100644 doc/administration/gitaly/faq.md create mode 100644 doc/administration/operations/extra_sidekiq_routing.md create mode 100644 doc/administration/troubleshooting/img/azure_configure_group_claim.png create mode 100644 doc/api/group_protected_environments.md create mode 100644 doc/api/status_checks.md create mode 100644 doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md create mode 100644 doc/ci/runners/configure_runners.md create mode 100644 doc/ci/runners/runners_scope.md create mode 100644 doc/development/documentation/styleguide/word_list.md create mode 100644 doc/development/fe_guide/content_editor.md create mode 100644 doc/development/fe_guide/img/content_editor_highlevel_diagram.png create mode 100644 doc/development/real_time.md create mode 100644 doc/subscriptions/img/license-file.png create mode 100644 doc/subscriptions/img/license-overview.png create mode 100644 doc/subscriptions/img/publicly-visible.png create mode 100644 doc/subscriptions/img/support-diagram.png create mode 100644 doc/topics/build_your_application.md create mode 100644 doc/topics/plan_and_track.md create mode 100644 doc/topics/release_your_application.md create mode 100644 doc/topics/set_up_organization.md create mode 100644 doc/topics/use_gitlab.md create mode 100644 doc/user/admin_area/analytics/img/admin_devops_adoption_v14_0.png delete mode 100644 doc/user/admin_area/settings/img/admin_required_pipeline.png create mode 100644 doc/user/admin_area/settings/img/continuous_integration_shared_runner_details_v14_0.png delete mode 100644 doc/user/admin_area/settings/img/custom_sign_in_page_v13_6.png create mode 100644 doc/user/application_security/dast/img/dast_auth_browser_scan_highlight.png create mode 100644 doc/user/application_security/dast/img/dast_auth_browser_scan_search_elements.png create mode 100644 doc/user/application_security/dast/img/dast_auth_report.jpg delete mode 100644 doc/user/application_security/img/unconfigured_security_approval_rules_and_enabled_jobs_v13_4.png delete mode 100644 doc/user/application_security/img/unconfigured_security_approval_rules_and_jobs_v13_4.png delete mode 100644 doc/user/application_security/vulnerabilities/img/vulnerability_page_merge_request_button_dropdown_v13_1.png delete mode 100644 doc/user/clusters/img/fluentd_v13_0.png create mode 100644 doc/user/clusters/management_project_template.md create mode 100644 doc/user/clusters/migrating_from_gma_to_project_template.md delete mode 100644 doc/user/group/devops_adoption/img/group_devops_adoption_v13_11.png create mode 100644 doc/user/group/devops_adoption/img/group_devops_adoption_v14_0.png delete mode 100644 doc/user/group/epics/img/epic_board_v13_10.png create mode 100644 doc/user/group/epics/img/epic_board_v14_0.png create mode 100644 doc/user/group/saml_sso/img/member_enterprise_badge_v14_0.png create mode 100644 doc/user/group/saml_sso/img/saml_group_links_v13_9.png create mode 100644 doc/user/packages/infrastructure_registry/index.md create mode 100644 doc/user/packages/terraform_module_registry/index.md create mode 100644 doc/user/packages/workflows/working_with_monorepos.md delete mode 100644 doc/user/profile/img/unknown_sign_in_email_v13_1.png create mode 100644 doc/user/profile/img/unknown_sign_in_email_v14_0.png delete mode 100644 doc/user/project/clusters/protect/web_application_firewall/img/guide_waf_ingress_disabled_settings_v12_10.png delete mode 100644 doc/user/project/clusters/protect/web_application_firewall/img/guide_waf_ingress_installation_v12_10.png delete mode 100644 doc/user/project/clusters/protect/web_application_firewall/img/guide_waf_ingress_save_changes_v12_10.png delete mode 100644 doc/user/project/clusters/protect/web_application_firewall/index.md delete mode 100644 doc/user/project/clusters/protect/web_application_firewall/quick_start_guide.md delete mode 100644 doc/user/project/img/code_owners_approval_new_protected_branch_v13_10.png delete mode 100644 doc/user/project/img/code_owners_approval_protected_branch_v13_10.png delete mode 100644 doc/user/project/integrations/hipchat.md create mode 100644 doc/user/project/issues/img/issue_type_change_v13_12.png delete mode 100644 doc/user/project/members/img/access_requests_management_v13_9.png delete mode 100644 doc/user/project/members/img/add_user_email_accept_v13_9.png delete mode 100644 doc/user/project/members/img/add_user_email_ready_v13_8.png delete mode 100644 doc/user/project/members/img/add_user_email_search_v13_8.png delete mode 100644 doc/user/project/members/img/withdraw_access_request_button.png create mode 100644 doc/user/project/merge_requests/commits.md delete mode 100644 doc/user/project/merge_requests/img/allow_collaboration.png delete mode 100644 doc/user/project/merge_requests/img/allow_collaboration_after_save.png create mode 100644 doc/user/project/merge_requests/img/code_quality_mr_diff_report_v14.png create mode 100644 doc/user/project/merge_requests/img/commit-button_v13_12.png create mode 100644 doc/user/project/merge_requests/img/conflict_ui_v14_0.png create mode 100644 doc/user/project/merge_requests/img/merge_request_pipeline.png create mode 100644 doc/user/project/merge_requests/img/project_merge_requests_list_view_v13_5.png delete mode 100644 doc/user/project/merge_requests/img/reviewer_approval_rules_form_v13_8.png delete mode 100644 doc/user/project/merge_requests/img/reviewer_approval_rules_sidebar_v13_8.png create mode 100644 doc/user/project/merge_requests/img/status_checks_branches_selector_v14_0.png create mode 100644 doc/user/project/merge_requests/img/status_checks_create_form_v14_0.png create mode 100644 doc/user/project/merge_requests/img/status_checks_delete_modal_v14_0.png create mode 100644 doc/user/project/merge_requests/img/status_checks_list_view_v14_0.png create mode 100644 doc/user/project/merge_requests/img/status_checks_update_form_v14_0.png delete mode 100644 doc/user/project/merge_requests/reviews/img/merge_request_pipeline.png delete mode 100644 doc/user/project/merge_requests/reviews/img/project_merge_requests_list_view_v13_5.png create mode 100644 doc/user/project/merge_requests/reviews/img/reviewer_approval_rules_form_v13_8.png create mode 100644 doc/user/project/merge_requests/reviews/img/reviewer_approval_rules_sidebar_v13_8.png create mode 100644 doc/user/project/merge_requests/status_checks.md create mode 100644 doc/user/project/merge_requests/widgets.md delete mode 100644 doc/user/project/repository/img/download_source_code.png delete mode 100644 doc/user/project/repository/img/file_ext_icons_repo_v12_10.png create mode 100644 doc/user/project/wiki/img/content_editor_v14.0.png create mode 100644 doc/user/project/wiki/img/use_new_editor_button_v14.0.png (limited to 'doc') diff --git a/doc/.vale/gitlab/Acronyms.yml b/doc/.vale/gitlab/Acronyms.yml index 01dafe45223..d7f5c750ec0 100644 --- a/doc/.vale/gitlab/Acronyms.yml +++ b/doc/.vale/gitlab/Acronyms.yml @@ -23,18 +23,18 @@ exceptions: - AWS - BSD - CAS + - CDN - CLI - CNA - CNAME - CORE - - CVS - - FREE - CPU - CRIME - CSRF - CSS - CSV - CVE + - CVS - DAG - DAST - DHCP @@ -43,15 +43,18 @@ exceptions: - DSA - DVCS - ECDSA + - ECS - EFS - EKS - EOL - EXIF - FAQ + - FIFO - FIPS - FOSS - FQDN - FREE + - FTP - GCP - GDK - GDPR @@ -61,8 +64,10 @@ exceptions: - GNU - GPG - GPL + - GPU - GUI - HAML + - HDD - HEAD - HIPAA - HTML @@ -80,6 +85,7 @@ exceptions: - JPEG - JPG - JSON + - JVM - JWT - LAN - LDAP @@ -92,11 +98,13 @@ exceptions: - MIT - MVC - NAT + - NDA - NFS - NGINX - NOTE - NTP - ONLY + - OSS - OWASP - PAT - PCI-DSS @@ -106,8 +114,10 @@ exceptions: - PGP - PHP - PNG + - POSIX - POST - PUT + - RAID - RAM - RDP - REST @@ -119,15 +129,17 @@ exceptions: - RSA - RSS - RVM - - SAML - SAAS + - SAML - SAST + - SATA - SCIM - SCP - SCSS - SDK - SELF - SEO + - SFTP - SHA - SLA - SMS @@ -162,6 +174,7 @@ exceptions: - UUID - VCS - VPC + - VPN - WIP - WSL - XML diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 61fd0148fd8..b38427ace09 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -19,3 +19,15 @@ swap: info: information repo: repository utilize: use + owner access: the Owner role + owner permission: the Owner role + owner permissions: the Owner role + maintainer access: the Maintainer role + maintainer permission: the Maintainer role + maintainer permissions: the Maintainer role + administrator access: the Administrator role + administrator permission: the Administrator role + administrator permissions: the Administrator role + developer access: the Developer role + developer permission: the Developer role + developer permissions: the Developer role diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index b31bead3bcd..99d2eb1f11a 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -4,7 +4,7 @@ # Checks for misused terms that should never be used at GitLab. # SubstitutionWarning.yml and SubstitionSuggestions.yml also exist. # -# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +# For a list of all options, see https://docs.errata.ai/vale/styles extends: substitution message: 'Use "%s" instead of "%s".' link: https://about.gitlab.com/handbook/communication/#top-misused-terms diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 05d0e5d4b85..d465767049f 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -13,6 +13,7 @@ anonymization anonymized Ansible Anthos +Apdex approvers architected architecting @@ -207,6 +208,7 @@ formatters Fugit fuzzer Gantt +Gemfile Gemnasium Gemojione Getter @@ -281,6 +283,7 @@ jQuery jsdom Jsonnet JupyterHub +Kaminari kanban kanbans kaniko @@ -292,7 +295,6 @@ keytab keytabs Kibana Kinesis -Klar Knative Kramdown Kroki @@ -439,6 +441,7 @@ Pritaly Priyanka profiler Prometheus +ProseMirror protobuf protobufs proxied @@ -546,6 +549,7 @@ serializer serializers serializing serverless +severities sharded sharding shfmt @@ -639,6 +643,7 @@ timeboxed timeboxes timeboxing timecop +tiptap todos tokenizer Tokenizers diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index a7f4fc10655..f0c4d947668 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Audit Events **(PREMIUM)** -GitLab offers a way to view the changes made within the GitLab server for owners and administrators on a [paid plan](https://about.gitlab.com/pricing/). +GitLab offers a way to view the changes made within the GitLab server for owners and administrators +on a [paid plan](https://about.gitlab.com/pricing/). GitLab system administrators can also take advantage of the logs located on the file system. See [the logs system documentation](logs.md#audit_jsonlog) for more details. @@ -49,10 +50,18 @@ When a user is being [impersonated](../user/admin_area/index.md#user-impersonati ### Group events **(PREMIUM)** -A user with a Owner role (or above) can retrieve group audit events of all users. -A user with a Developer or Maintainer role is limited to group audit events based on their individual actions. +A user with: + +- Owner role (or above) can retrieve group audit events of all users. +- Developer or Maintainer role is limited to group audit events based on their individual actions. + +Group events do not include project audit events. + +To view a group's audit events: + +1. Go to the group. +1. On the left sidebar, select **Security & Compliance > Audit Events**. -To view a group's audit events, navigate to **Group > Security & Compliance > Audit Events**. From there, you can see the following actions: - Group name or path changed. @@ -82,7 +91,11 @@ Group events can also be accessed via the [Group Audit Events API](../api/audit_ A user with a Maintainer role (or above) can retrieve project audit events of all users. A user with a Developer role is limited to project audit events based on their individual actions. -To view a project's audit events, navigate to **Project > Security & Compliance > Audit Events**. +To view a project's audit events: + +1. Go to the project. +1. On the left sidebar, select **Security & Compliance > Audit Events**. + From there, you can see the following actions: - Added or removed deploy keys @@ -120,10 +133,14 @@ Server-wide audit events introduce the ability to observe user actions across the entire instance of your GitLab server, making it easy to understand who changed what and when for audit purposes. -To view the server-wide administrator log, visit **Admin Area > Monitoring > Audit Events**. +Instance events do not include group or project audit events. -In addition to the group and project events, the following user actions are also -recorded: +To view the server-wide audit events: + +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Monitoring > Audit Events**. + +The following user actions are recorded: - Sign-in events and the authentication type (such as standard, LDAP, or OmniAuth) - Failed sign-ins @@ -147,6 +164,17 @@ recorded: Instance events can also be accessed via the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). +### Sign-in events **(FREE)** + +Successful sign-in events are the only Audit Events available at all tiers. To see +successful sign-in events: + +1. Select your avatar. +1. Select **Edit profile > Authentication log**. + +After upgrading from GitLab Free to a paid tier, successful sign-in events are the only Audit +Events visible in Audit Events views until more events are logged. + ### Missing events Some events are not tracked in Audit Events. See the following @@ -171,7 +199,7 @@ It may make the user interface for your project or audit events very busy, and t `audit_events` PostgreSQL table may increase considerably. It's disabled by default to prevent performance degradations on GitLab instances with very high Git write traffic. -In an upcoming release, Audit Events for Git push events will be enabled +In an upcoming release, Audit Events for Git push events are planned to be enabled by default. Follow our [Partitioning strategy for Audit Events epic](https://gitlab.com/groups/gitlab-org/-/epics/3206) for updates. If you still wish to enable **Repository push** events in your instance, follow @@ -213,11 +241,12 @@ Export to CSV allows customers to export the current filter view of your audit e CSV file, which stores tabular data in plain text. The data provides a comprehensive view with respect to audit events. -To export the Audit Events to CSV, navigate to -**{monitor}** **Admin Area > Monitoring > Audit Events** +To export the Audit Events to CSV: +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Monitoring > Audit Events**. 1. Select the available search [filters](#search). -1. Click **Export as CSV**. +1. Select **Export as CSV**. ### Sort diff --git a/doc/administration/audit_reports.md b/doc/administration/audit_reports.md index 2721ee39b60..6fa592b96db 100644 --- a/doc/administration/audit_reports.md +++ b/doc/administration/audit_reports.md @@ -5,10 +5,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w description: 'Learn how to create evidence artifacts typically requested by a 3rd party auditor.' --- -# Audit Reports +# Audit reports **(FREE)** GitLab can help owners and administrators respond to auditors by generating -comprehensive reports. These **Audit Reports** vary in scope, depending on the +comprehensive reports. These audit reports vary in scope, depending on the needs. ## Use cases diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 364c7cebea3..bc6a854c518 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -7,7 +7,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # General LDAP setup **(FREE SELF)** -GitLab integrates with LDAP to support user authentication. +GitLab integrates with [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) +to support user authentication. This integration works with most LDAP-compliant directory servers, including: @@ -20,58 +21,48 @@ This integration works with most LDAP-compliant directory servers, including: Users added through LDAP take a [licensed seat](../../../subscriptions/self_managed/index.md#billable-users). GitLab Enterprise Editions (EE) include enhanced integration, -including group membership syncing as well as multiple LDAP servers support. - -## Overview - -[LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) -stands for **Lightweight Directory Access Protocol**, which is a standard -application protocol for accessing and maintaining distributed directory -information services over an Internet Protocol (IP) network. +including group membership syncing and multiple LDAP server support. ## Security GitLab assumes that LDAP users: - Are not able to change their LDAP `mail`, `email`, or `userPrincipalName` attributes. - An LDAP user who is allowed to change their email on the LDAP server can potentially + An LDAP user allowed to change their email on the LDAP server can potentially [take over any account](#enabling-ldap-sign-in-for-existing-gitlab-users) on your GitLab server. -- Have unique email addresses, otherwise it is possible for LDAP users with the same +- Have unique email addresses. If not, it's possible for LDAP users with the same email address to share the same GitLab account. We recommend against using LDAP integration if your LDAP users are -allowed to change their 'mail', 'email' or 'userPrincipalName' attribute on -the LDAP server or share email addresses. +allowed to change their `mail`, `email` or `userPrincipalName` attributes on +the LDAP server, or share email addresses. ### User deletion -If a user is deleted from the LDAP server, they are also blocked in GitLab. -Users are immediately blocked from logging in. However, there is an -LDAP check cache time of one hour (see note) which means users that -are already logged in or are using Git over SSH are be able to access -GitLab for up to one hour. Manually block the user in the GitLab Admin Area to -immediately block all access. - -GitLab Enterprise Edition Premium supports a -[configurable sync time](#adjusting-ldap-user-sync-schedule). **(PREMIUM)** +Users deleted from the LDAP server are immediately blocked from signing in +to GitLab. However, there's an LDAP check cache time of one hour (which is +[configurable](#adjusting-ldap-user-sync-schedule) for GitLab Premium users). +This means users already signed-in or who are using Git over SSH can access +GitLab for up to one hour. Manually block the user in the GitLab Admin Area +to immediately block all access. ## Git password authentication -LDAP-enabled users can always authenticate with Git using their GitLab username -or email and LDAP password, even if password authentication for Git is disabled +LDAP-enabled users can authenticate with Git using their GitLab username or +email and LDAP password, even if password authentication for Git is disabled in the application settings. ## Enabling LDAP sign-in for existing GitLab users -When a user signs in to GitLab with LDAP for the first time, and their LDAP -email address is the primary email address of an existing GitLab user, then -the LDAP DN is associated with the existing user. If the LDAP email -attribute is not found in the GitLab user database, a new user is created. +When a user signs in to GitLab with LDAP for the first time and their LDAP +email address is the primary email address of an existing GitLab user, the +LDAP DN is associated with the existing user. If the LDAP email attribute +isn't found in the GitLab user database, a new user is created. In other words, if an existing GitLab user wants to enable LDAP sign-in for themselves, they should check that their GitLab email address matches their -LDAP email address, and then sign into GitLab via their LDAP credentials. +LDAP email address, and then sign into GitLab by using their LDAP credentials. ## Google Secure LDAP @@ -95,7 +86,8 @@ NOTE: The `encryption` value `simple_tls` corresponds to 'Simple TLS' in the LDAP library. `start_tls` corresponds to StartTLS, not to be confused with regular TLS. Normally, if you specify `simple_tls` it is on port 636, while `start_tls` (StartTLS) -would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced with `start_tls` and `ssl` was replaced with `simple_tls`. +would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced +with `start_tls` and `ssl` was replaced with `simple_tls`. LDAP users must have a set email address, regardless of whether or not it's used to sign in. @@ -165,23 +157,23 @@ production: ### Basic Configuration Settings -| Setting | Description | Required | Examples | -| ------- | ----------- | -------- | -------- | -| `label` | A human-friendly name for your LDAP server. It is displayed on your sign-in page. | yes | `'Paris'` or `'Acme, Ltd.'` | -| `host` | IP address or domain name of your LDAP server. | yes | `'ldap.mydomain.com'` | -| `port` | The port to connect with on your LDAP server. Always an integer, not a string. | yes | `389` or `636` (for SSL) | -| `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | -| `bind_dn` | The full DN of the user you bind with. | no | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | -| `password` | The password of the bind user. | no | `'your_great_password'` | -| `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | yes | `'start_tls'` or `'simple_tls'` or `'plain'` | -| `verify_certificates` | Enables SSL certificate verification if encryption method is `start_tls` or `simple_tls`. Defaults to true. | no | boolean | -| `timeout` | Set a timeout, in seconds, for LDAP queries. This helps avoid blocking a request if the LDAP server becomes unresponsive. A value of `0` means there is no timeout. (default: `10`) | no | `10` or `30` | -| `active_directory` | This setting specifies if LDAP server is Active Directory LDAP server. For non-AD servers it skips the AD specific queries. If your LDAP server is not AD, set this to false. | no | boolean | -| `allow_username_or_email_login` | If enabled, GitLab ignores everything after the first `@` in the LDAP username submitted by the user on sign-in. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you need to disable this setting, because the userPrincipalName contains an `@`. | no | boolean | -| `block_auto_created_users` | To maintain tight control over the number of billable users on your GitLab installation, enable this setting to keep new users blocked until they have been cleared by an administrator (default: false). | no | boolean | -| `base` | Base where we can search for users. | yes | `'ou=people,dc=gitlab,dc=example'` or `'DC=mydomain,DC=com'` | -| `user_filter` | Filter LDAP users. Format: [RFC 4515](https://tools.ietf.org/search/rfc4515) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | no | For examples, read [Examples of user filters](#examples-of-user-filters). | -| `lowercase_usernames` | If lowercase_usernames is enabled, GitLab converts the name to lower case. | no | boolean | +| Setting | Description | Required | Examples | +|--------------------|-------------|----------|----------| +| `label` | A human-friendly name for your LDAP server. It is displayed on your sign-in page. | **{check-circle}** Yes | `'Paris'` or `'Acme, Ltd.'` | +| `host` | IP address or domain name of your LDAP server. | **{check-circle}** Yes | `'ldap.mydomain.com'` | +| `port` | The port to connect with on your LDAP server. Always an integer, not a string. | **{check-circle}** Yes | `389` or `636` (for SSL) | +| `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | **{check-circle}** Yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | +| `bind_dn` | The full DN of the user you bind with. | **{dotted-circle}** No | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | +| `password` | The password of the bind user. | **{dotted-circle}** No | `'your_great_password'` | +| `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | **{check-circle}** Yes | `'start_tls'` or `'simple_tls'` or `'plain'` | +| `verify_certificates` | Enables SSL certificate verification if encryption method is `start_tls` or `simple_tls`. Defaults to true. | **{dotted-circle}** No | boolean | +| `timeout` | Set a timeout, in seconds, for LDAP queries. This helps avoid blocking a request if the LDAP server becomes unresponsive. A value of `0` means there is no timeout. (default: `10`) | **{dotted-circle}** No | `10` or `30` | +| `active_directory` | This setting specifies if LDAP server is Active Directory LDAP server. For non-AD servers it skips the AD specific queries. If your LDAP server is not AD, set this to false. | **{dotted-circle}** No | boolean | +| `allow_username_or_email_login` | If enabled, GitLab ignores everything after the first `@` in the LDAP username submitted by the user on sign-in. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you need to disable this setting, because the userPrincipalName contains an `@`. | **{dotted-circle}** No | boolean | +| `block_auto_created_users` | To maintain tight control over the number of billable users on your GitLab installation, enable this setting to keep new users blocked until they have been cleared by an administrator (default: false). | **{dotted-circle}** No | boolean | +| `base` | Base where we can search for users. | **{check-circle}** Yes | `'ou=people,dc=gitlab,dc=example'` or `'DC=mydomain,DC=com'` | +| `user_filter` | Filter LDAP users. Format: [RFC 4515](https://tools.ietf.org/search/rfc4515) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | **{dotted-circle}** No | For examples, read [Examples of user filters](#examples-of-user-filters). | +| `lowercase_usernames` | If enabled, GitLab converts the name to lower case. | **{dotted-circle}** No | boolean | #### Examples of user filters @@ -192,41 +184,44 @@ Some examples of the `user_filter` field syntax: ### SSL Configuration Settings -| Setting | Description | Required | Examples | -| ------- | ----------- | -------- | -------- | -| `ca_file` | Specifies the path to a file containing a PEM-format CA certificate, for example, if you need to use an internal CA. | no | `'/etc/ca.pem'` | -| `ssl_version` | Specifies the SSL version for OpenSSL to use, if the OpenSSL default is not appropriate. | no | `'TLSv1_1'` | -| `ciphers` | Specific SSL ciphers to use in communication with LDAP servers. | no | `'ALL:!EXPORT:!LOW:!aNULL:!eNULL:!SSLv2'` | -| `cert` | Client certificate | no | `'-----BEGIN CERTIFICATE----- -----END CERTIFICATE -----'` | -| `key` | Client private key | no | `'-----BEGIN PRIVATE KEY----- -----END PRIVATE KEY -----'` | +| Setting | Description | Required | Examples | +|---------------|-------------|----------|----------| +| `ca_file` | Specifies the path to a file containing a PEM-format CA certificate, for example, if you need to use an internal CA. | **{dotted-circle}** No | `'/etc/ca.pem'` | +| `ssl_version` | Specifies the SSL version for OpenSSL to use, if the OpenSSL default is not appropriate. | **{dotted-circle}** No | `'TLSv1_1'` | +| `ciphers` | Specific SSL ciphers to use in communication with LDAP servers. | **{dotted-circle}** No | `'ALL:!EXPORT:!LOW:!aNULL:!eNULL:!SSLv2'` | +| `cert` | Client certificate. | **{dotted-circle}** No | `'-----BEGIN CERTIFICATE----- -----END CERTIFICATE -----'` | +| `key` | Client private key. | **{dotted-circle}** No | `'-----BEGIN PRIVATE KEY----- -----END PRIVATE KEY -----'` | ### Attribute Configuration Settings -LDAP attributes that GitLab uses to create an account for the LDAP user. The specified attribute can either be the attribute name as a string (for example, `'mail'`), or an array of attribute names to try in order (for example, `['mail', 'email']`). Note that the user's LDAP sign-in is the attribute specified as `uid` above. +LDAP attributes that GitLab uses to create an account for the LDAP user. The specified +attribute can either be the attribute name as a string (for example, `'mail'`), or an +array of attribute names to try in order (for example, `['mail', 'email']`). Note that +the user's LDAP sign-in is the attribute specified as `uid` above. -| Setting | Description | Required | Examples | -| ------- | ----------- | -------- | -------- | -| `username` | The username is used in paths for the user's own projects (like `gitlab.example.com/username/project`) and when mentioning them in issues, merge request and comments (like `@username`). If the attribute specified for `username` contains an email address, the GitLab username is part of the email address before the `@`. | no | `['uid', 'userid', 'sAMAccountName']` | -| `email` | LDAP attribute for user email. | no | `['mail', 'email', 'userPrincipalName']` | -| `name` | LDAP attribute for user display name. If `name` is blank, the full name is taken from the `first_name` and `last_name`. | no | Attributes `'cn'`, or `'displayName'` commonly carry full names. Alternatively, you can force the use of `first_name` and `last_name` by specifying an absent attribute such as `'somethingNonExistent'`. | -| `first_name` | LDAP attribute for user first name. Used when the attribute configured for `name` does not exist. | no | `'givenName'` | -| `last_name` | LDAP attribute for user last name. Used when the attribute configured for `name` does not exist. | no | `'sn'` | +| Setting | Description | Required | Examples | +|--------------|-------------|----------|----------| +| `username` | The username is used in paths for the user's own projects (like `gitlab.example.com/username/project`) and when mentioning them in issues, merge request and comments (like `@username`). If the attribute specified for `username` contains an email address, the GitLab username is part of the email address before the `@`. | **{dotted-circle}** No | `['uid', 'userid', 'sAMAccountName']` | +| `email` | LDAP attribute for user email. | **{dotted-circle}** No | `['mail', 'email', 'userPrincipalName']` | +| `name` | LDAP attribute for user display name. If `name` is blank, the full name is taken from the `first_name` and `last_name`. | **{dotted-circle}** No | Attributes `'cn'`, or `'displayName'` commonly carry full names. Alternatively, you can force the use of `first_name` and `last_name` by specifying an absent attribute such as `'somethingNonExistent'`. | +| `first_name` | LDAP attribute for user first name. Used when the attribute configured for `name` does not exist. | **{dotted-circle}** No | `'givenName'` | +| `last_name` | LDAP attribute for user last name. Used when the attribute configured for `name` does not exist. | **{dotted-circle}** No | `'sn'` | ### LDAP Sync Configuration Settings **(PREMIUM SELF)** -| Setting | Description | Required | Examples | -| ------- | ----------- | -------- | -------- | -| `group_base` | Base used to search for groups. | no | `'ou=groups,dc=gitlab,dc=example'` | -| `admin_group` | The CN of a group containing GitLab administrators. Note: Not `cn=administrators` or the full DN. | no | `'administrators'` | -| `external_groups` | An array of CNs of groups containing users that should be considered external. Note: Not `cn=interns` or the full DN. | no | `['interns', 'contractors']` | -| `sync_ssh_keys` | The LDAP attribute containing a user's public SSH key. | no | `'sshPublicKey'` or false if not set | +| Setting | Description | Required | Examples | +|-------------------|-------------|----------|----------| +| `group_base` | Base used to search for groups. | **{dotted-circle}** No | `'ou=groups,dc=gitlab,dc=example'` | +| `admin_group` | The CN of a group containing GitLab administrators. Note: Not `cn=administrators` or the full DN. | **{dotted-circle}** No | `'administrators'` | +| `external_groups` | An array of CNs of groups containing users that should be considered external. Note: Not `cn=interns` or the full DN. | **{dotted-circle}** No | `['interns', 'contractors']` | +| `sync_ssh_keys` | The LDAP attribute containing a user's public SSH key. | **{dotted-circle}** No | `'sshPublicKey'` or false if not set | ### Set up LDAP user filter If you want to limit all GitLab access to a subset of the LDAP users on your LDAP server, the first step should be to narrow the configured `base`. However, -it is sometimes necessary to filter users further. In this case, you can set up -an LDAP user filter. The filter must comply with +it's sometimes necessary to further filter users. In this case, you can set +up an LDAP user filter. The filter must comply with [RFC 4515](https://tools.ietf.org/search/rfc4515). **Omnibus configuration** @@ -252,7 +247,7 @@ production: ``` If you want to limit access to the nested members of an Active Directory -group, you can use the following syntax: +group, use the following syntax: ```plaintext (memberOf:1.2.840.113556.1.4.1941:=CN=My Group,DC=Example,DC=com) @@ -260,11 +255,10 @@ group, you can use the following syntax: For more information about this "LDAP_MATCHING_RULE_IN_CHAIN" filter, see the following [Microsoft Search Filter Syntax](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) document. -Support for nested members in the user filter should not be confused with +Support for nested members in the user filter shouldn't be confused with [group sync nested groups support](#supported-ldap-group-typesattributes). **(PREMIUM SELF)** -Please note that GitLab does not support the custom filter syntax used by -OmniAuth LDAP. +GitLab does not support the custom filter syntax used by OmniAuth LDAP. #### Escaping special characters @@ -342,7 +336,7 @@ an alternative such as SAML is preferred. This allows LDAP to be used for group sync, while also allowing your SAML identity provider to handle additional checks like custom 2FA. -When LDAP web sign in is disabled, users don't see an **LDAP** tab on the sign in page. +When LDAP web sign in is disabled, users don't see an **LDAP** tab on the sign-in page. This does not disable [using LDAP credentials for Git access](#git-password-authentication). **Omnibus configuration** @@ -373,7 +367,7 @@ Instead of having the LDAP integration credentials stored in plaintext in the co use an encrypted file for the LDAP credentials. To use this feature, you first need to enable [GitLab encrypted configuration](../../encrypted_configuration.md). -The encrypted configuration for LDAP exists in an encrypted YAML file. By default the file will be created at +The encrypted configuration for LDAP exists in an encrypted YAML file. By default the file is created at `shared/encrypted_configuration/ldap.yaml.enc`. This location is configurable in the GitLab configuration. The unencrypted contents of the file should be a subset of the secret settings from your `servers` block in the LDAP @@ -520,7 +514,9 @@ gitlab_rails['ldap_servers'] = { } ``` -If you configure multiple LDAP servers, use a unique naming convention for the `label` section of each entry. That label is used as the display name of the tab shown on the sign-in page. +If you configure multiple LDAP servers, use a unique naming convention for the +`label` section of each entry. That label is used as the display name of the tab +shown on the sign-in page. ## User sync **(PREMIUM SELF)** @@ -545,13 +541,13 @@ For more information, see [Bitmask Searches in LDAP](https://ctovswild.com/2009/ The user is set to an `ldap_blocked` state in GitLab if the previous conditions -fail. This means the user is not able to sign in or push/pull code. +fail. This means the user cannot sign in or push or pull code. The process also updates the following user information: -- Email address. -- If `sync_ssh_keys` is set, SSH public keys. -- If Kerberos is enabled, Kerberos identity. +- Email address +- SSH public keys (if `sync_ssh_keys` is set) +- Kerberos identity (if Kerberos is enabled) The LDAP sync process: @@ -643,19 +639,22 @@ or more LDAP group links](#adding-group-links). ### Adding group links **(PREMIUM SELF)** -For information on adding group links via CNs and filters, refer to [the GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap). +For information on adding group links by using CNs and filters, refer to the +[GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap). ### Administrator sync **(PREMIUM SELF)** As an extension of group sync, you can automatically manage your global GitLab administrators. Specify a group CN for `admin_group` and all members of the -LDAP group will be given administrator privileges. The configuration looks +LDAP group are given administrator privileges. The configuration looks like the following. NOTE: Administrators are not synced unless `group_base` is also specified alongside `admin_group`. Also, only specify the CN of the `admin_group`, as opposed to the full DN. +Additionally, note that if an LDAP user has an `admin` role, but is not a member of the `admin_group` +group, GitLab revokes their `admin` role when syncing. **Omnibus configuration** @@ -705,8 +704,10 @@ When enabled, the following applies: To enable it you need to: 1. [Enable LDAP](#configuration) -1. Go to **Admin Area > Settings > Visibility and access controls**. -1. Make sure the **Lock memberships to LDAP synchronization** checkbox is selected. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility and access controls** section. +1. Ensure the **Lock memberships to LDAP synchronization** checkbox is selected. ### Adjusting LDAP group sync schedule **(PREMIUM SELF)** @@ -717,13 +718,13 @@ The values shown are in cron format. If needed, you can use a WARNING: Do not start the sync process too frequently as this could lead to multiple syncs running concurrently. This is primarily a concern -for installations with a large number of LDAP users. Please review the +for installations with a large number of LDAP users. Review the [LDAP group sync benchmark metrics](#benchmarks) to see how your installation compares before proceeding. You can manually configure LDAP group sync times by setting the following configuration values. The example below shows how to set group -sync to run once every 2 hours at the top of the hour. +sync to run once every two hours at the top of the hour. **Omnibus installations** @@ -749,7 +750,7 @@ sync to run once every 2 hours at the top of the hour. ### External groups **(PREMIUM SELF)** -Using the `external_groups` setting will allow you to mark all users belonging +Using the `external_groups` setting allows you to mark all users belonging to these groups as [external users](../../../user/permissions.md#external-users). Group membership is checked periodically through the `LdapGroupSync` background task. @@ -786,15 +787,14 @@ task. ### Group sync technical details -There is a lot going on with group sync 'under the hood'. This section -outlines what LDAP queries are executed and what behavior you can expect -from group sync. +This section outlines what LDAP queries are executed and what behavior you +can expect from group sync. Group member access are downgraded from a higher level if their LDAP group -membership changes. For example, if a user has 'Owner' rights in a group and the -next group sync reveals they should only have 'Developer' privileges, their +membership changes. For example, if a user the Owner role in a group and the +next group sync reveals they should only have the Developer role, their access is adjusted accordingly. The only exception is if the user is the -*last* owner in a group. Groups need at least one owner to fulfill +last owner in a group. Groups need at least one owner to fulfill administrative duties. #### Supported LDAP group types/attributes @@ -805,18 +805,20 @@ GitLab supports LDAP groups that use member attributes: - `submember` - `uniquemember` - `memberof` -- `memberuid`. +- `memberuid` + +This means group sync supports (at least) LDAP groups with the following object +classes: -This means group sync supports, at least, LDAP groups with the following object classes: -`groupOfNames`, `posixGroup`, and `groupOfUniqueNames`. +- `groupOfNames` +- `posixGroup` +- `groupOfUniqueNames` -Other object classes should work fine as long as members -are defined as one of the mentioned attributes. This also means GitLab supports -Microsoft Active Directory, Apple Open Directory, Open LDAP, and 389 Server. -Other LDAP servers should work, too. +Other object classes should work if members are defined as one of the +mentioned attributes. -Active Directory also supports nested groups. Group sync recursively -resolves membership if `active_directory: true` is set in the configuration file. +Active Directory supports nested groups. Group sync recursively resolves +membership if `active_directory: true` is set in the configuration file. ##### Nested group memberships @@ -842,7 +844,7 @@ Group sync was written to be as performant as possible. Data is cached, database queries are optimized, and LDAP queries are minimized. The last benchmark run revealed the following metrics: -For 20000 LDAP users, 11000 LDAP groups and 1000 GitLab groups with 10 +For 20,000 LDAP users, 11,000 LDAP groups, and 1,000 GitLab groups with 10 LDAP group links each: - Initial sync (no existing members assigned in GitLab) took 1.8 hours @@ -855,4 +857,4 @@ network and LDAP server response time affects these metrics. ## Troubleshooting -Please see our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). +See our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 1e6684751ed..acafe52007b 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -20,7 +20,7 @@ or `encryption: 'simple_tls'` and `port: 636`. #### Connection times out -If GitLab cannot reach your LDAP endpoint, you will see a message like this: +If GitLab cannot reach your LDAP endpoint, you see a message like this: ```plaintext Could not authenticate you from Ldapmain because "Connection timed out - user specified timeout". @@ -79,7 +79,7 @@ adapter.ldap_search(options) ``` For examples of how this is run, -[review the `Adapter` module](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb). +[review the `Adapter` module](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb). ### User sign-ins @@ -145,7 +145,8 @@ may see the following message: `Access denied for your LDAP account`. We have a workaround, based on toggling the access level of affected users: -1. As an administrator, go to **Admin Area > Overview > Users**. +1. As an administrator, on the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Overview > Users**. 1. Select the name of the affected user. 1. In the user's administrative page, press **Edit** on the top right of the page. 1. Change the user's access level from `Regular` to `Admin` (or vice versa), @@ -192,6 +193,24 @@ This shows you which user has this email address. One of two steps must be taken The user can do either of these steps [in their profile](../../../user/profile/index.md#access-your-user-profile) or an administrator can do it. +#### Projects limit errors + +The following errors indicate that a limit or restriction is activated, but an associated data +field contains no data: + +- `Projects limit can't be blank`. +- `Projects limit is not a number`. + +To resolve this: + +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, go to **Settings > General**. +1. Expand both of the following: + - **Account and limit**. + - **Sign-up restrictions**. +1. Check, for example, the **Default projects limit** or **Allowed domains for sign-ups** + fields and ensure that a relevant value is configured. + #### Debug LDAP user filter [`ldapsearch`](#ldapsearch) allows you to test your configured @@ -329,8 +348,9 @@ things to check to debug the situation. group](index.md#adding-group-links). - Check that the user has an LDAP identity: 1. Sign in to GitLab as an administrator user. - 1. Go to **Admin area > Users**. - 1. Search for the user + 1. On the top bar, select **Menu >** **{admin}** **Admin**. + 1. On the left sidebar, select **Overview > Users**. + 1. Search for the user. 1. Open the user by clicking their name. Do not click **Edit**. 1. Select the **Identities** tab. There should be an LDAP identity with an LDAP DN as the 'Identifier'. If not, this user hasn't signed in with @@ -367,7 +387,7 @@ the following are true: - The configured `admin_group` in the `gitlab.rb` is a CN, rather than a DN or an array. - This CN falls under the scope of the configured `group_base`. - The members of the `admin_group` have already signed into GitLab with their LDAP - credentials. GitLab will only grant this administrator access to the users whose + credentials. GitLab only grants this administrator access to the users whose accounts are already connected to LDAP. If all the above are true and the users are still not getting access, [run a manual @@ -396,7 +416,7 @@ output](#example-console-output-after-a-group-sync). ##### Example console output after a group sync **(PREMIUM SELF)** Like the output from the user sync, the output from the [manual group -sync](#sync-all-groups) will also be very verbose. However, it contains lots +sync](#sync-all-groups) is also very verbose. However, it contains lots of helpful information. Indicates the point where syncing actually begins: @@ -407,7 +427,7 @@ Started syncing 'ldapmain' provider for 'my_group' group The following entry shows an array of all user DNs GitLab sees in the LDAP server. Note that these are the users for a single LDAP group, not a GitLab group. If -you have multiple LDAP groups linked to this GitLab group, you will see multiple +you have multiple LDAP groups linked to this GitLab group, you see multiple log entries like this - one for each LDAP group. If you don't see an LDAP user DN in this log entry, LDAP is not returning the user when we do the lookup. Verify the user is actually in the LDAP group. @@ -421,7 +441,7 @@ Members in 'ldap_group_1' LDAP group: ["uid=john0,ou=people,dc=example,dc=com", "uid=mary4,ou=people,dc=example,dc=com"] ``` -Shortly after each of the above entries, you will see a hash of resolved member +Shortly after each of the above entries, you see a hash of resolved member access levels. This hash represents all user DNs GitLab thinks should have access to this group, and at which access level (role). This hash is additive, and more DNs may be added, or existing entries modified, based on additional @@ -462,21 +482,21 @@ Finally, the following entry says syncing has finished for this group: Finished syncing all providers for 'my_group' group ``` -Once all the configured group links have been synchronized, GitLab will look +Once all the configured group links have been synchronized, GitLab looks for any Administrators or External users to sync: ```shell Syncing admin users for 'ldapmain' provider ``` -The output will look similar to what happens with a single group, and then -this line will indicate the sync is finished: +The output looks similar to what happens with a single group, and then +this line indicates the sync is finished: ```shell Finished syncing admin users for 'ldapmain' provider ``` -If [administrator sync](index.md#administrator-sync) is not configured, you'll see a message +If [administrator sync](index.md#administrator-sync) is not configured, you see a message stating as such: ```shell @@ -502,8 +522,8 @@ group = Group.find_by(name: 'my_gitlab_group') EE::Gitlab::Auth::Ldap::Sync::Group.execute_all_providers(group) ``` -The output will be similar to -[that you'd get from syncing all groups](#example-console-output-after-a-group-sync). +The output is similar to +[that you get from syncing all groups](#example-console-output-after-a-group-sync). #### Query a group in LDAP **(PREMIUM SELF)** @@ -524,24 +544,25 @@ ldap_group.member_uids When an LDAP user is created in GitLab, their LDAP DN is stored for later reference. -If GitLab cannot find a user by their DN, it will fall back -to finding the user by their email. If the lookup is successful, GitLab will -update the stored DN to the new value so both values will now match what's in +If GitLab cannot find a user by their DN, it falls back +to finding the user by their email. If the lookup is successful, GitLab +updates the stored DN to the new value so both values now match what's in LDAP. -If the email has changed and the DN has not, GitLab will find the user with +If the email has changed and the DN has not, GitLab finds the user with the DN and update its own record of the user's email to match the one in LDAP. -However, if the primary email _and_ the DN change in LDAP, then GitLab will -have no way of identifying the correct LDAP record of the user and, as a -result, the user will be blocked. To rectify this, the user's existing -profile will have to be updated with at least one of the new values (primary +However, if the primary email _and_ the DN change in LDAP, then GitLab +has no way of identifying the correct LDAP record of the user and, as a +result, the user is blocked. To rectify this, the user's existing +profile must be updated with at least one of the new values (primary email or DN) so the LDAP record can be found. -The following script will update the emails for all provided users so they -won't be blocked or unable to access their accounts. +The following script updates the emails for all provided users so they +aren't blocked or unable to access their accounts. ->**NOTE**: The following script will require that any new accounts with the new +NOTE: +The following script requires that any new accounts with the new email address are removed first. This is because emails have to be unique in GitLab. Go to the [rails console](#rails-console) and then run: @@ -588,23 +609,23 @@ users, [see what to do when no users are found](#no-users-are-found). ### GitLab logs If a user account is blocked or unblocked due to the LDAP configuration, a -message will be [logged to `application.log`](../../logs.md#applicationlog). +message is [logged to `application.log`](../../logs.md#applicationlog). If there is an unexpected error during an LDAP lookup (configuration error, -timeout), the sign-in is rejected and a message will be [logged to +timeout), the sign-in is rejected and a message is [logged to `production.log`](../../logs.md#productionlog). ### ldapsearch -`ldapsearch` is a utility that will allow you to query your LDAP server. You can +`ldapsearch` is a utility that allows you to query your LDAP server. You can use it to test your LDAP settings and ensure that the settings you're using -will get you the results you expect. +get you the results you expect. When using `ldapsearch`, be sure to use the same settings you've already specified in your `gitlab.rb` configuration so you can confirm what happens when those exact settings are used. -Running this command on the GitLab host will also help confirm that there's no +Running this command on the GitLab host also helps confirm that there's no obstruction between the GitLab host and LDAP. For example, consider the following GitLab configuration: @@ -685,9 +706,9 @@ For instructions about how to use the rails console, refer to this #### Enable debug output -This will provide debug output that will be useful to see -what GitLab is doing and with what. This value is not persisted, and will only -be enabled for this session in the rails console. +This provides debug output that is useful to see +what GitLab is doing and with what. This value is not persisted, and is only +enabled for this session in the rails console. To enable debug output in the rails console, [enter the rails console](#rails-console) and run: diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index 88e9180b103..64b42339d19 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -1,5 +1,6 @@ --- redirect_to: '../../integration/saml.md' +remove_date: '2021-06-15' --- This document was moved to [another location](../../integration/saml.md). diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index 1b9638411de..8e5c162001e 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -127,5 +127,5 @@ time="2020-10-29T04:44:14Z" level=warning msg="Config: failed to fetch" agent_id It means that the path to the configuration project is incorrect, or the path to `config.yaml` inside the project is not valid. -To fix this, ensure that the paths to the configuration repo and to the `config.yaml` file +To fix this, ensure that the paths to the configuration repository and to the `config.yaml` file are correct. diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 470dc1b4f9e..6b80ddbcdb5 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -4,7 +4,7 @@ group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Compliance features +# Compliance features **(FREE)** You can configure the following GitLab features to help ensure that your GitLab instance meets common compliance standards. Click a feature name for additional @@ -13,20 +13,20 @@ documentation. The [security features](../security/README.md) in GitLab may also help you meet relevant compliance standards. -|Feature |GitLab tier |GitLab SaaS | Product level | -| ---------| :--------: | :-------: | :-----------: | -|**[Restrict SSH Keys](../security/ssh_keys_restrictions.md)**
Control the technology and key length of SSH keys used to access GitLab|Free+||Instance| -|**[Granular user roles and flexible permissions](../user/permissions.md)**
Manage access and permissions with five different user roles and settings for external users. Set permissions according to people's role, rather than either read or write access to a repository. Don't share the source code with people that only need access to the issue tracker.|Free+|✓|Instance, Group, Project| -|**[Enforce TOS acceptance](../user/admin_area/settings/terms.md)**
Enforce your users accepting new terms of service by blocking GitLab traffic.|Free+||Instance| -|**[Email all users of a project, group, or entire server](../tools/email.md)**
An administrator can email groups of users based on project or group membership, or email everyone using the GitLab instance. This is great for scheduled maintenance or upgrades.|Premium+||Instance| -|**[Omnibus package supports log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding)**
Forward your logs to a central system.|Premium+||Instance| -|**[Lock project membership to group](../user/group/index.md#prevent-members-from-being-added-to-a-group)**
Group owners can prevent new members from being added to projects within a group.|Premium+|✓|Group| -|**[LDAP group sync](auth/ldap/index.md#group-sync)**
GitLab Enterprise Edition gives administrators the ability to automatically sync groups and manage SSH keys, permissions, and authentication, so you can focus on building your product, not configuring your tools.|Premium+||Instance| -|**[LDAP group sync filters](auth/ldap/index.md#group-sync)**
GitLab Enterprise Edition Premium gives more flexibility to synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions.|Premium+||Instance| -|**[Audit events](audit_events.md)**
To maintain the integrity of your code, GitLab Enterprise Edition Premium gives administrators the ability to view any modifications made within the GitLab server in an advanced audit events system, so you can control, analyze, and track every change.|Premium+|✓|Instance, Group, Project| -|**[Auditor users](auditor_users.md)**
Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance.|Premium+||Instance| -|**[Credentials inventory](../user/admin_area/credentials_inventory.md)**
With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. |Ultimate||Instance| -|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners) and [custom CI Configuration Paths](../ci/pipelines/settings.md#custom-cicd-configuration-path)**
GitLab Premium users can leverage the GitLab cross-project YAML configurations to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles.|Premium+|✓|Project| -|**[Compliance frameworks](../user/project/settings/index.md#compliance-frameworks)**
Create a custom compliance framework at the group level to describe the type of compliance requirements any child project needs to follow. |Premium+|✓|Group| -|**[Compliance pipelines](../user/project/settings/index.md#compliance-pipeline-configuration)**
Define a pipeline configuration to run for any projects with a given compliance framework.|Ultimate|✓|Group| -|**[Compliance dashboard](../user/compliance/compliance_dashboard/index.md)**
Quickly get visibility into the compliance posture of your organization.|Ultimate|✓|Group| +| Feature | GitLab tier | GitLab SaaS | Product level | +|----------|:-----------:|:-----------:|:-------------:| +|**[Restrict SSH Keys](../security/ssh_keys_restrictions.md)**
Control the technology and key length of SSH keys used to access GitLab. | Free+ | **{dotted-circle}** No | Instance | +|**[Granular user roles and flexible permissions](../user/permissions.md)**
Manage access and permissions with five different user roles and settings for external users. Set permissions according to people's role, rather than either read or write access to a repository. Don't share the source code with people that only need access to the issue tracker. | Free+ | **{check-circle}** Yes | Instance, Group, Project | +|**[Enforce TOS acceptance](../user/admin_area/settings/terms.md)**
Enforce your users accepting new terms of service by blocking GitLab traffic. | Free+ | **{dotted-circle}** No | Instance | +|**[Email all users of a project, group, or entire server](../tools/email.md)**
An administrator can email groups of users based on project or group membership, or email everyone using the GitLab instance. This is great for scheduled maintenance or upgrades. | Premium+ | **{dotted-circle}** No | Instance | +|**[Omnibus package supports log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding)**
Forward your logs to a central system. | Premium+ | **{dotted-circle}** No | Instance | +|**[Lock project membership to group](../user/group/index.md#prevent-members-from-being-added-to-a-group)**
Group owners can prevent new members from being added to projects within a group. | Premium+ | **{check-circle}** Yes | Group | +|**[LDAP group sync](auth/ldap/index.md#group-sync)**
GitLab Enterprise Edition gives administrators the ability to automatically sync groups and manage SSH keys, permissions, and authentication, so you can focus on building your product, not configuring your tools. | Premium+ | **{dotted-circle}** No | Instance | +|**[LDAP group sync filters](auth/ldap/index.md#group-sync)**
GitLab Enterprise Edition Premium gives more flexibility to synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions. | Premium+ | **{dotted-circle}** No | Instance | +|**[Audit events](audit_events.md)**
To maintain the integrity of your code, GitLab Enterprise Edition Premium gives administrators the ability to view any modifications made within the GitLab server in an advanced audit events system, so you can control, analyze, and track every change. | Premium+ | **{check-circle}** Yes | Instance, Group, Project | +|**[Auditor users](auditor_users.md)**
Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance. | Premium+ | **{dotted-circle}** No | Instance | +|**[Credentials inventory](../user/admin_area/credentials_inventory.md)**
With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. | Ultimate | **{dotted-circle}** No | Instance | +|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners) and [custom CI Configuration Paths](../ci/pipelines/settings.md#custom-cicd-configuration-file)**
GitLab Premium users can leverage the GitLab cross-project YAML configurations to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles. | Premium+ | **{check-circle}** Yes | Project | +|**[Compliance frameworks](../user/project/settings/index.md#compliance-frameworks)**
Create a custom compliance framework at the group level to describe the type of compliance requirements any child project needs to follow. | Premium+ | **{check-circle}** Yes | Group | +|**[Compliance pipelines](../user/project/settings/index.md#compliance-pipeline-configuration)**
Define a pipeline configuration to run for any projects with a given compliance framework. | Ultimate | **{check-circle}** Yes | Group | +|**[Compliance dashboard](../user/compliance/compliance_dashboard/index.md)**
Quickly get visibility into the compliance posture of your organization. | Ultimate | **{check-circle}** Yes | Group | diff --git a/doc/administration/configure.md b/doc/administration/configure.md new file mode 100644 index 00000000000..12a8f721ccf --- /dev/null +++ b/doc/administration/configure.md @@ -0,0 +1,16 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +--- + +# Configure your GitLab installation + +Customize and configure your self-managed GitLab installation. + +- [Authentication](auth/README.md) +- [Configuration](../user/admin_area/index.md) +- [Repository storage](repository_storage_paths.md) +- [Geo](geo/index.md) +- [Packages](packages/index.md) diff --git a/doc/administration/consul.md b/doc/administration/consul.md index a748259aff0..c88047c4c61 100644 --- a/doc/administration/consul.md +++ b/doc/administration/consul.md @@ -15,11 +15,17 @@ turn communicate with the servers. GitLab Premium includes a bundled version of [Consul](https://www.consul.io/) a service networking solution that you can manage by using `/etc/gitlab/gitlab.rb`. +## Prerequisites + +Before configuring Consul: + +1. Review the [reference architecture](reference_architectures/index.md#available-reference-architectures) + documentation to determine the number of Consul server nodes you should have. +1. If necessary, ensure the [appropriate ports are open](https://docs.gitlab.com/omnibus/package-information/defaults.html#ports) in your firewall. + ## Configure the Consul nodes -After you review the [reference architecture](reference_architectures/index.md#available-reference-architectures) -documentation to determine the number of Consul server nodes you should have, -on _each_ Consul server node: +On _each_ Consul server node: 1. Follow the instructions to [install](https://about.gitlab.com/install/) GitLab by choosing your preferred platform, but do not supply the @@ -80,6 +86,15 @@ within each node. The command will return an empty array if the cluster is healt curl "http://127.0.0.1:8500/v1/health/state/critical" ``` +If the Consul version has changed, you'll see a notice at the end of `gitlab-ctl reconfigure` +informing you that Consul needs to be restarted for the new version to be used. + +Restart Consul one node at a time: + +```shell +sudo gitlab-ctl restart consul +``` + Consul nodes communicate using the raft protocol. If the current leader goes offline, there needs to be a leader election. A leader node must exist to facilitate synchronization across the cluster. If too many nodes go offline at the same time, diff --git a/doc/administration/database_load_balancing.md b/doc/administration/database_load_balancing.md index bd34a82f688..9c1ed9b3477 100644 --- a/doc/administration/database_load_balancing.md +++ b/doc/administration/database_load_balancing.md @@ -4,9 +4,10 @@ group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Database Load Balancing **(PREMIUM SELF)** +# Database Load Balancing **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1283) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1283) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60894) from GitLab Premium to GitLab Free in 14.0. Distribute read-only queries among multiple database servers. @@ -21,8 +22,6 @@ component may increase reliability and availability through redundancy. When database load balancing is enabled in GitLab, the load is balanced using a simple round-robin algorithm, without any external dependencies such as Redis. -Load balancing is not enabled for Sidekiq as this would lead to consistency -problems, and Sidekiq mostly performs writes anyway. In the following image, you can see the load is balanced rather evenly among all the secondaries (`db4`, `db5`, `db6`). Because `SELECT` queries are not @@ -105,6 +104,32 @@ the following. This will balance the load between `host1.example.com` and 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. +### Enable the load balancer for Sidekiq + +Sidekiq mostly writes to the database, which means that most of its traffic hits the +primary database. + +Some background jobs can use database replicas to read application state. +This allows to offload the primary database. + +Load balancing is disabled by default in Sidekiq. When enabled, we can define +[the data consistency](../development/sidekiq_style_guide.md#job-data-consistency) +requirements for a specific job. + +To enable it, define the `ENABLE_LOAD_BALANCING_FOR_SIDEKIQ` variable to the environment, as shown below. + +For Omnibus installations: + +```ruby +gitlab_rails['env'] = {"ENABLE_LOAD_BALANCING_FOR_SIDEKIQ" => "true"} +``` + +For installations from source: + +```shell +export ENABLE_LOAD_BALANCING_FOR_SIDEKIQ="true" +``` + ## Service Discovery > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/5883) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. @@ -171,28 +196,6 @@ Some nameservers (like [Consul](https://www.consul.io/docs/discovery/dns#udp-bas queried over UDP. To overcome this issue, you can use TCP for querying by setting `use_tcp` to `true`. -### Forking - -NOTE: -Starting with GitLab 13.0, Puma is the default web server used in GitLab -all-in-one package based installations as well as GitLab Helm chart deployments. - -If you use an application server that forks, such as Unicorn, you _have to_ -update your Unicorn configuration to start service discovery _after_ a fork. -Failure to do so leads to service discovery only running in the parent -process. If you are using Unicorn, then you can add the following to your -Unicorn configuration file: - -```ruby -after_fork do |server, worker| - defined?(Gitlab::Database::LoadBalancing) && - Gitlab::Database::LoadBalancing.start_service_discovery -end -``` - -This ensures that service discovery is started in both the parent and all -child processes. - ## Balancing queries Read-only `SELECT` queries balance among all the secondary hosts. diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index a168584e754..057abce0ed5 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Environment variables +# Environment variables **(FREE SELF)** GitLab exposes certain environment variables which can be used to override their defaults values. @@ -32,8 +32,6 @@ You can use the following environment variables to override certain values: | `GITLAB_HOST` | string | The full URL of the GitLab server (including `http://` or `https://`). | | `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation. | | `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners. | -| `GITLAB_UNICORN_MEMORY_MAX` | integer | The maximum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | -| `GITLAB_UNICORN_MEMORY_MIN` | integer | The minimum memory threshold (in bytes) for the [unicorn-worker-killer](operations/unicorn.md#unicorn-worker-killer). | | `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | | `UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`). | diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index 89543e446ac..9fc65fdd0b5 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference, howto --- @@ -17,7 +17,7 @@ data as payload. The response code from the external service determines if GitLa should accept or reject the pipeline. If the response is: - `200`, the pipeline is accepted. -- `4XX`, the pipeline is rejected. +- `406`, the pipeline is rejected. - Other codes, the pipeline is accepted and logged. If there's an error or the request times out, the pipeline is accepted. @@ -74,7 +74,9 @@ required number of seconds. "id": { "type": "integer" }, "username": { "type": "string" }, "email": { "type": "string" }, - "created_at": { "type": ["string", "null"], "format": "date-time" } + "created_at": { "type": ["string", "null"], "format": "date-time" }, + "current_sign_in_ip": { "type": ["string", "null"] }, + "last_sign_in_ip": { "type": ["string", "null"] } } }, "pipeline": { @@ -126,6 +128,17 @@ required number of seconds. "plan": { "type": "string" }, "trial": { "type": "boolean" } } + }, + "provisioning_group": { + "type": "object", + "required": [ + "plan", + "trial" + ], + "properties": { + "plan": { "type": "string" }, + "trial": { "type": "boolean" } + } } } } diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index 9ba50cfbf2e..44abf4a875d 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -127,6 +127,5 @@ Feature.disabled?(:my_awesome_feature) => false ``` -When the feature is ready, GitLab will remove the feature flag, the option for -enabling and disabling it will no longer exist, and the feature will become -available in all instances. +When the feature is ready, GitLab removes the feature flag, and the option for +enabling and disabling it no longer exists. The feature becomes available in all instances. diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index c60f0040496..f73c961f541 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -33,7 +33,7 @@ see the [system hooks](../system_hooks/system_hooks.md) documentation. The file hooks must be placed directly into the `file_hooks` directory, subdirectories are ignored. There is an -[`example` directory inside `file_hooks`](https://gitlab.com/gitlab-org/gitlab/tree/master/file_hooks/examples) +[`example` directory inside `file_hooks`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/file_hooks/examples) where you can find some basic examples. Follow the steps below to set up a custom hook: @@ -63,8 +63,11 @@ need to restart GitLab to apply a new file hook. If a file hook executes with non-zero exit code or GitLab fails to execute it, a message is logged to: -- `gitlab-rails/plugin.log` in an Omnibus installation. -- `log/plugin.log` in a source installation. +- `gitlab-rails/file_hook.log` in an Omnibus installation. +- `log/file_hook.log` in a source installation. + +NOTE: +Before 14.0 release, the file name was `plugin.log` ## Creating file hooks @@ -79,7 +82,7 @@ require 'json' require 'mail' # The incoming variables are in JSON format so we need to parse it first. -ARGS = JSON.parse(STDIN.read) +ARGS = JSON.parse($stdin.read) # We only want to trigger this file hook on the event project_create return unless ARGS['event_name'] == 'project_create' diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 7c6f4a32b57..f6f88e9b193 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -7,17 +7,14 @@ type: howto # Disaster Recovery (Geo) **(PREMIUM SELF)** -Geo replicates your database, your Git repositories, and few other assets. -We will support and replicate more data in the future, that will enable you to -failover with minimal effort, in a disaster situation. - -See [Geo limitations](../index.md#limitations) for more information. +Geo replicates your database, your Git repositories, and few other assets, +but there are some [limitations](../index.md#limitations). WARNING: Disaster recovery for multi-secondary configurations is in **Alpha**. For the latest updates, check the [Disaster Recovery epic for complete maturity](https://gitlab.com/groups/gitlab-org/-/epics/3574). Multi-secondary configurations require the complete re-synchronization and re-configuration of all non-promoted secondaries and -will cause downtime. +causes downtime. ## Promoting a **secondary** Geo node in single-secondary configurations @@ -91,13 +88,16 @@ Note the following when promoting a secondary: before proceeding. If the secondary node [has been paused](../../geo/index.md#pausing-and-resuming-replication), the promotion performs a point-in-time recovery to the last known state. - Data that was created on the primary while the secondary was paused will be lost. + Data that was created on the primary while the secondary was paused is lost. - A new **secondary** should not be added at this time. If you want to add a new **secondary**, do this after you have completed the entire process of promoting the **secondary** to the **primary**. - If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` error message during this process, for more information, see this [troubleshooting advice](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). +- If you run into errors when using `--force` or `--skip-preflight-checks` before 13.5 during this process, + for more information, see this + [troubleshooting advice](../replication/troubleshooting.md#errors-when-using---skip-preflight-checks-or---force). #### Promoting a **secondary** node running on a single machine @@ -243,6 +243,7 @@ required: sets the database to read-write. The instructions vary depending on where your database is hosted: - [Amazon RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) - [Azure PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) + - [Google Cloud SQL](https://cloud.google.com/sql/docs/mysql/replication/manage-replicas#promote-replica) - For other external PostgreSQL databases, save the following script in your secondary node, for example `/tmp/geo_promote.sh`, and modify the connection parameters to match your environment. Then, execute it to promote the replica: @@ -493,7 +494,7 @@ must disable the **primary** site: WARNING: If the secondary site [has been paused](../../geo/index.md#pausing-and-resuming-replication), this performs a point-in-time recovery to the last known state. -Data that was created on the primary while the secondary was paused will be lost. +Data that was created on the primary while the secondary was paused is lost. 1. SSH in to the database node in the **secondary** and trigger PostgreSQL to promote to read-write: @@ -509,7 +510,7 @@ Data that was created on the primary while the secondary was paused will be lost `geo_secondary_role`: NOTE: - Depending on your architecture these steps will need to be run on any GitLab node that is external to the **secondary** Kubernetes cluster. + Depending on your architecture, these steps need to run on any GitLab node that is external to the **secondary** Kubernetes cluster. ```ruby ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. @@ -537,13 +538,13 @@ Data that was created on the primary while the secondary was paused will be lost 1. Update the existing cluster configuration. - You can retrieve the existing config with Helm: + You can retrieve the existing configuration with Helm: ```shell helm --namespace gitlab get values gitlab-geo > gitlab.yaml ``` - The existing config will contain a section for Geo that should resemble: + The existing configuration contains a section for Geo that should resemble: ```yaml geo: @@ -560,9 +561,9 @@ Data that was created on the primary while the secondary was paused will be lost To promote the **secondary** cluster to a **primary** cluster, update `role: secondary` to `role: primary`. - You can remove the entire `psql` section if the cluster will remain as a primary site, this refers to the tracking database and will be ignored whilst the cluster is acting as a primary site. + If the cluster remains as a primary site, you can remove the entire `psql` section; it refers to the tracking database and is ignored whilst the cluster is acting as a primary site. - Update the cluster with the new config: + Update the cluster with the new configuration: ```shell helm upgrade --install --version gitlab-geo gitlab/gitlab --namespace gitlab -f gitlab.yaml diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index bd8467f5437..d50078da172 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -35,7 +35,7 @@ required scheduled maintenance period significantly. A common strategy for keeping this period as short as possible for data stored in files is to use `rsync` to transfer the data. An initial `rsync` can be performed ahead of the maintenance window; subsequent `rsync`s (including a -final transfer inside the maintenance window) will then transfer only the +final transfer inside the maintenance window) then transfers only the *changes* between the **primary** node and the **secondary** nodes. Repository-centric strategies for using `rsync` effectively can be found in the @@ -50,7 +50,7 @@ this command reports `ERROR - Replication is not up-to-date` even if replication is actually up-to-date. This bug was fixed in GitLab 13.8 and later. -Run this command to list out all preflight checks and automatically check if replication and verification are complete before scheduling a planned failover to ensure the process will go smoothly: +Run this command to list out all preflight checks and automatically check if replication and verification are complete before scheduling a planned failover to ensure the process goes smoothly: ```shell gitlab-ctl promotion-preflight-checks @@ -73,7 +73,7 @@ In GitLab 12.4, you can optionally allow GitLab to manage replication of Object Database settings are automatically replicated to the **secondary** node, but the `/etc/gitlab/gitlab.rb` file must be set up manually, and differs between nodes. If features such as Mattermost, OAuth or LDAP integration are enabled -on the **primary** node but not the **secondary** node, they will be lost during failover. +on the **primary** node but not the **secondary** node, they are lost during failover. Review the `/etc/gitlab/gitlab.rb` file for both nodes and ensure the **secondary** node supports everything the **primary** node does **before** scheduling a planned failover. @@ -119,7 +119,7 @@ time to complete If any objects are failing to replicate, this should be investigated before scheduling the maintenance window. Following a planned failover, anything that -failed to replicate will be **lost**. +failed to replicate is **lost**. You can use the [Geo status API](../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) to review failed objects and the reasons for failure. @@ -136,9 +136,9 @@ This [content was moved to another location](background_verification.md). On the **primary** node, navigate to **Admin Area > Messages**, add a broadcast message. You can check under **Admin Area > Geo** to estimate how long it -will take to finish syncing. An example message would be: +takes to finish syncing. An example message would be: -> A scheduled maintenance will take place at XX:XX UTC. We expect it to take +> A scheduled maintenance takes place at XX:XX UTC. We expect it to take > less than 1 hour. ## Prevent updates to the **primary** node @@ -151,7 +151,7 @@ be disabled on the primary site: 1. Disable non-Geo periodic background jobs on the **primary** node by navigating to **Admin Area > Monitoring > Background Jobs > Cron**, pressing `Disable All`, and then pressing `Enable` for the `geo_sidekiq_cron_config_worker` cron job. - This job will re-enable several other cron jobs that are essential for planned + This job re-enables several other cron jobs that are essential for planned failover to complete successfully. ## Finish replicating and verifying all data @@ -161,7 +161,7 @@ be disabled on the primary site: 1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** and wait for all queues except those with `geo` in the name to drop to 0. These queues contain work that has been submitted by your users; failing over - before it is completed will cause the work to be lost. + before it is completed, causes the work to be lost. 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the following conditions to be true of the **secondary** node you are failing over to: @@ -176,15 +176,15 @@ be disabled on the primary site: to verify the integrity of CI artifacts, LFS objects, and uploads in file storage. -At this point, your **secondary** node will contain an up-to-date copy of everything the -**primary** node has, meaning nothing will be lost when you fail over. +At this point, your **secondary** node contains an up-to-date copy of everything the +**primary** node has, meaning nothing was lost when you fail over. ## Promote the **secondary** node Finally, follow the [Disaster Recovery docs](index.md) to promote the -**secondary** node to a **primary** node. This process will cause a brief outage on the **secondary** node, and users may need to log in again. +**secondary** node to a **primary** node. This process causes a brief outage on the **secondary** node, and users may need to log in again. -Once it is completed, the maintenance window is over! Your new **primary** node will now +Once it is completed, the maintenance window is over! Your new **primary** node, now begin to diverge from the old one. If problems do arise at this point, failing back to the old **primary** node [is possible](bring_primary_back.md), but likely to result in the loss of any data uploaded to the new **primary** in the meantime. diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 780e391973c..295a448c432 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -27,7 +27,7 @@ to clone and fetch large repositories, speeding up development. For a video introduction to Geo, see [Introduction to GitLab Geo - GitLab Features](https://www.youtube.com/watch?v=-HDLxSjEh6w). -To make sure you're using the right version of the documentation, navigate to [the Geo page on GitLab.com](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/geo/index.md) and choose the appropriate release from the **Switch branch/tag** dropdown. For example, [`v13.7.6-ee`](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.7.6-ee/doc/administration/geo/index.md). +To make sure you're using the right version of the documentation, navigate to [the Geo page on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/geo/index.md) and choose the appropriate release from the **Switch branch/tag** dropdown. For example, [`v13.7.6-ee`](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.7.6-ee/doc/administration/geo/index.md). Geo uses a set of defined terms that is described in the [Geo Glossary](glossary.md), please familiarize yourself with those terms. @@ -56,11 +56,12 @@ Geo provides: ### Gitaly Cluster Geo should not be confused with [Gitaly Cluster](../gitaly/praefect.md). For more information about -the difference between Geo and Gitaly Cluster, see [Gitaly Cluster compared to Geo](../gitaly/index.md#gitaly-cluster-compared-to-geo). +the difference between Geo and Gitaly Cluster, see +[How does Gitaly Cluster compare to Geo?](../gitaly/faq.md#how-does-gitaly-cluster-compare-to-geo). ## How it works -Your Geo instance can be used for cloning and fetching projects, in addition to reading any data. This will make working with large repositories over large distances much faster. +Your Geo instance can be used for cloning and fetching projects, in addition to reading any data. This makes working with large repositories over large distances much faster. ![Geo overview](replication/img/geo_overview.png) @@ -121,7 +122,7 @@ The following are required to run Geo: The following operating systems are known to ship with a current version of OpenSSH: - [CentOS](https://www.centos.org) 7.4+ - [Ubuntu](https://ubuntu.com) 16.04+ -- PostgreSQL 11+ with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) +- PostgreSQL 12+ with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) - Git 2.9+ - Git-lfs 2.4.2+ on the user side when using LFS - All sites must run the same GitLab version. @@ -150,17 +151,17 @@ NOTE: When using HTTP or HTTPS proxying, your load balancer must be configured to pass through the `Connection` and `Upgrade` hop-by-hop headers. See the [web terminal](../integration/terminal.md) integration guide for more details. NOTE: -When using HTTPS protocol for port 443, you will need to add an SSL certificate to the load balancers. +When using HTTPS protocol for port 443, you need to add an SSL certificate to the load balancers. If you wish to terminate SSL at the GitLab application server instead, use TCP protocol. ### LDAP -We recommend that if you use LDAP on your **primary** site, you also set up secondary LDAP servers on each **secondary** site. Otherwise, users will not be able to perform Git operations over HTTP(s) on the **secondary** site using HTTP Basic Authentication. However, Git via SSH and personal access tokens will still work. +We recommend that if you use LDAP on your **primary** site, you also set up secondary LDAP servers on each **secondary** site. Otherwise, users are unable to perform Git operations over HTTP(s) on the **secondary** site using HTTP Basic Authentication. However, Git via SSH and personal access tokens still works. NOTE: -It is possible for all **secondary** sites to share an LDAP server, but additional latency can be an issue. Also, consider what LDAP server will be available in a [disaster recovery](disaster_recovery/index.md) scenario if a **secondary** site is promoted to be a **primary** site. +It is possible for all **secondary** sites to share an LDAP server, but additional latency can be an issue. Also, consider what LDAP server is available in a [disaster recovery](disaster_recovery/index.md) scenario if a **secondary** site is promoted to be a **primary** site. -Check for instructions on how to set up replication in your LDAP service. Instructions will be different depending on the software or service used. For example, OpenLDAP provides [these instructions](https://www.openldap.org/doc/admin24/replication.html). +Check for instructions on how to set up replication in your LDAP service. Instructions are different depending on the software or service used. For example, OpenLDAP provides [these instructions](https://www.openldap.org/doc/admin24/replication.html). ### Geo Tracking Database @@ -179,9 +180,9 @@ This daemon: - Reads a log of events replicated by the **primary** site to the **secondary** database instance. - Updates the Geo Tracking Database instance with changes that need to be executed. -When something is marked to be updated in the tracking database instance, asynchronous jobs running on the **secondary** site will execute the required operations and update the state. +When something is marked to be updated in the tracking database instance, asynchronous jobs running on the **secondary** site execute the required operations and update the state. -This new architecture allows GitLab to be resilient to connectivity issues between the sites. It doesn't matter how long the **secondary** site is disconnected from the **primary** site as it will be able to replay all the events in the correct order and become synchronized with the **primary** site again. +This new architecture allows GitLab to be resilient to connectivity issues between the sites. It doesn't matter how long the **secondary** site is disconnected from the **primary** site as it is able to replay all the events in the correct order and become synchronized with the **primary** site again. ## Limitations @@ -196,7 +197,7 @@ This list of limitations only reflects the latest version of GitLab. If you are - Object pools for forked project deduplication work only on the **primary** site, and are duplicated on the **secondary** site. - GitLab Runners cannot register with a **secondary** site. Support for this is [planned for the future](https://gitlab.com/gitlab-org/gitlab/-/issues/3294). - Configuring Geo **secondary** sites to [use high-availability configurations of PostgreSQL](https://gitlab.com/groups/gitlab-org/-/epics/2536) is currently in **alpha** support. -- [Selective synchronization](replication/configuration.md#selective-synchronization) only limits what repositories are replicated. The entire PostgreSQL data is still replicated. Selective synchronization is not built to accomodate compliance / export control use cases. +- [Selective synchronization](replication/configuration.md#selective-synchronization) only limits what repositories are replicated. The entire PostgreSQL data is still replicated. Selective synchronization is not built to accommodate compliance / export control use cases. ### Limitations on replication/verification @@ -280,7 +281,7 @@ For an example of how to set up a location-aware Git remote URL with AWS Route53 ### Backfill -Once a **secondary** site is set up, it will start replicating missing data from +Once a **secondary** site is set up, it starts replicating missing data from the **primary** site in a process known as **backfill**. You can monitor the synchronization process on each Geo site from the **primary** site's **Geo Nodes** dashboard in your browser. diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index e2f12cbd8dc..a1461a64518 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -47,8 +47,8 @@ verification methods: | Blobs | Container registry _(file system)_ | Geo with API/Docker API | _Not implemented_ | | Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _Not implemented_ | | Blobs | Package registry _(file system)_ | Geo with API | SHA256 checksum | -| Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | SHA256 checksum | -| Blobs | Versioned Terraform State _(file system)_ | Geo with API | _Not implemented_ | +| Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | Versioned Terraform State _(file system)_ | Geo with API | SHA256 checksum | | Blobs | Versioned Terraform State _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | External Merge Request Diffs _(file system)_ | Geo with API | _Not implemented_ | | Blobs | External Merge Request Diffs _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | @@ -180,7 +180,7 @@ successfully, you must replicate their data using some other means. |[Project wiki repository](../../../user/project/wiki/) | **Yes** (10.2) | **Yes** (10.7) | No | | |[Group wiki repository](../../../user/project/wiki/index.md#group-wikis) | [**Yes** (13.10)](https://gitlab.com/gitlab-org/gitlab/-/issues/208147) | No | No | Behind feature flag `geo_group_wiki_repository_replication`, enabled by default. | |[Uploads](../../uploads.md) | **Yes** (10.2) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1817) | No | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both sites and comparing the output between them. | -|[LFS objects](../../lfs/index.md) | **Yes** (10.2) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Via Object Storage provider if supported. Native Geo support (Beta). | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both sites and comparing the output between them. GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). | +|[LFS objects](../../lfs/index.md) | **Yes** (10.2) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8922) | Via Object Storage provider if supported. Native Geo support (Beta). | Verified only on transfer or manually using [Integrity Check Rake Task](../../raketasks/check.md) on both sites and comparing the output between them. GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696).

Behind feature flag `geo_lfs_object_replication`, enabled by default. | |[Personal snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | No | | |[Project snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | No | | |[CI job artifacts (other than Job Logs)](../../../ci/pipelines/job_artifacts.md) | **Yes** (10.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923) | Via Object Storage provider if supported. Native Geo support (Beta). | Verified only manually using [Integrity Check Rake Task](../../raketasks/check.md) on both sites and comparing the output between them. | @@ -189,49 +189,25 @@ successfully, you must replicate their data using some other means. |[Object pools for forked project deduplication](../../../development/git_object_deduplication.md) | **Yes** | No | No | | |[Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | No | Disabled by default. See [instructions](docker_registry.md) to enable. | |[Content in object storage (beta)](object_storage.md) | **Yes** (12.4) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/13845) | No | | -|[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | Via Object Storage provider if supported. Native Geo support (Beta). | | -|[Package Registry for npm](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for Maven](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for Conan](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for NuGet](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for PyPI](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for Composer](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Package Registry for generic packages](../../../user/packages/generic_packages/index.md) | **Yes** (13.5) | **Yes** (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | -|[Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_terraform_state_version_replication`, enabled by default. | -|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_merge_request_diff_replication`, enabled by default. | +|[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | No | Designs also require replication of LFS objects and Uploads. | +|[Package Registry for npm](../../../user/packages/npm_registry/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for Maven](../../../user/packages/maven_repository/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for Conan](../../../user/packages/conan_repository/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for NuGet](../../../user/packages/nuget_repository/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for PyPI](../../../user/packages/pypi_repository/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for Composer](../../../user/packages/composer_repository/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Package Registry for generic packages](../../../user/packages/generic_packages/index.md) | **Yes** (13.5) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.12) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_terraform_state_version_replication`, enabled by default. Verification was behind the feature flag `geo_terraform_state_version_verification`, which was removed in 14.0| +|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification is under development, behind the feature flag `geo_merge_request_diff_verification`, introduced in 14.0.| |[Versioned snippets](../../../user/snippets.md#versioned-snippets) | [**Yes** (13.7)](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [No](https://gitlab.com/groups/gitlab-org/-/epics/2810) | No | | |[Server-side Git hooks](../../server_hooks.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | | |[Elasticsearch integration](../../../integration/elasticsearch.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | | -|[GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | No | | +|[GitLab Pages](../../pages/index.md) | [No](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | Via Object Storage provider if supported. **No** native Geo support (Beta). | | |[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | Blocked on [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Note that replication of this cache is not needed for Disaster Recovery purposes because it can be recreated from external sources. | -|[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Not planned because they are ephemeral and sensitive. They can be regenerated on demand. | +|[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | | Not planned because they are ephemeral and sensitive. They can be regenerated on demand. | -#### LFS object replication using the self service framework +#### Limitation of verification for files in Object Storage -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276696) in GitLab 13.12. -> - [Deployed behind a feature flag](../../../user/feature_flags.md), enabled by default. -> - Not enabled on GitLab.com as Geo is not enabled. -> - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-lfs-object-replication-using-the-self-service-framework). +GitLab managed Object Storage replication support [is in beta](object_storage.md#enabling-gitlab-managed-object-storage-replication). -There can be [risks when disabling released features](../../../user/feature_flags.md#risks-when-disabling-released-features). -Refer to this feature's version history for more details. - -##### Enable or disable LFS object replication using the self service framework - -LFS object replication using the self service framework is under development but ready for production use. It is -deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:geo_lfs_object_replication) -``` - -To disable it: - -```ruby -Feature.disable(:geo_lfs_object_replication) -``` +Locally stored files are verified but remote stored files are not. diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index a83a1c22db6..ef41b2ff172 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -17,13 +17,13 @@ On each **secondary** site, there is a read-only replicated copy of the GitLab d A **secondary** site also has a tracking database where it stores which projects have been synced. Geo compares the two databases to find projects that are not yet tracked. -At the start, this tracking database is empty, so Geo will start trying to update from every project that it can see in the GitLab database. +At the start, this tracking database is empty, so Geo tries to update from every project that it can see in the GitLab database. For each project to sync: -1. Geo will issue a `git fetch geo --mirror` to get the latest information from the **primary** site. - If there are no changes, the sync will be fast and end quickly. Otherwise, it will pull the latest commits. -1. The **secondary** site will update the tracking database to store the fact that it has synced projects A, B, C, etc. +1. Geo issues a `git fetch geo --mirror` to get the latest information from the **primary** site. + If there are no changes, the sync is fast. Otherwise, it has to pull the latest commits. +1. The **secondary** site updates the tracking database to store the fact that it has synced projects A, B, C, etc. 1. Repeat until all projects are synced. When someone pushes a commit to the **primary** site, it generates an event in the GitLab database that the repository has changed. @@ -70,4 +70,4 @@ Yes. See [Docker Registry for a **secondary** site](docker_registry.md). ## Can I login to a secondary site? -Yes, but secondary sites receive all authentication data (like user accounts and logins) from the primary instance. This means you will be re-directed to the primary for authentication and routed back afterwards. +Yes, but secondary sites receive all authentication data (like user accounts and logins) from the primary instance. This means you are re-directed to the primary for authentication and then routed back. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index 8f67e70c9e2..c6b1078ddf0 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -43,7 +43,7 @@ The following are GitLab upgrade validation tests we performed. - Outcome: Partial success because we observed downtime during the upgrade of the primary and secondary sites. - Follow up issues/actions: - [Fix zero-downtime upgrade process/instructions for multi-node Geo deployments](https://gitlab.com/gitlab-org/gitlab/-/issues/225684) - - [Geo:check Rake task: Exclude AuthorizedKeysCommand check if node not running Puma/Unicorn](https://gitlab.com/gitlab-org/gitlab/-/issues/225454) + - [Geo:check Rake task: Exclude AuthorizedKeysCommand check if node not running Puma](https://gitlab.com/gitlab-org/gitlab/-/issues/225454) - [Update instructions in the next upgrade issue to include monitoring HAProxy dashboards](https://gitlab.com/gitlab-org/gitlab/-/issues/225359) [Upgrade Geo multi-node installation](https://gitlab.com/gitlab-org/gitlab/-/issues/208104): @@ -53,7 +53,7 @@ The following are GitLab upgrade validation tests we performed. - Outcome: Partial success because we did not run the looping pipeline during the demo to validate zero-downtime. - Follow up issues: - - [Clarify how Puma/Unicorn should include deploy node](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5460) + - [Clarify how Puma should include deploy node](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5460) - [Investigate MR creation failure after upgrade to 12.9.10](https://gitlab.com/gitlab-org/gitlab/-/issues/223282) Closed as false positive. ### February 2020 diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 59bb3884a02..ea2488b65fb 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -17,9 +17,9 @@ described, it is possible to adapt these instructions to your needs. _[diagram source - GitLab employees only](https://docs.google.com/drawings/d/1z0VlizKiLNXVVVaERFwgsIOuEgjcUqDTWPdQYsE7Z4c/edit)_ -The topology above assumes that the **primary** and **secondary** Geo clusters +The topology above assumes the **primary** and **secondary** Geo clusters are located in two separate locations, on their own virtual network -with private IP addresses. The network is configured such that all machines within +with private IP addresses. The network is configured such that all machines in one geographic location can communicate with each other using their private IP addresses. The IP addresses given are examples and may be different depending on the network topology of your deployment. @@ -44,9 +44,10 @@ Support for PostgreSQL on **secondary** nodes in multi-node configuration Because of the additional complexity involved in setting up this configuration for PostgreSQL and Redis, it is not covered by this Geo multi-node documentation. -For more information about setting up a multi-node PostgreSQL cluster and Redis cluster using the omnibus package see the multi-node documentation for -[PostgreSQL](../../postgresql/replication_and_failover.md) and -[Redis](../../redis/replication_and_failover.md), respectively. +For more information on setting up a multi-node PostgreSQL cluster and Redis cluster using the Omnibus GitLab package, see: + +- [PostgreSQL multi-node documentation](../../postgresql/replication_and_failover.md) +- [Redis multi-node documentation](../../redis/replication_and_failover.md) NOTE: It is possible to use cloud hosted services for PostgreSQL and Redis, but this is beyond the scope of this document. @@ -60,8 +61,8 @@ you already have a working GitLab instance that is in-use, it can be used as a The second cluster serves as the **secondary** node. Again, use the [GitLab multi-node documentation](../../reference_architectures/index.md) to set this up. -It's a good idea to log in and test it, however, note that its data is -wiped out as part of the process of replicating from the **primary**. +It's a good idea to log in and test it. However, be aware that its data is +wiped out as part of the process of replicating from the **primary** node. ## Configure the GitLab cluster to be the **primary** node @@ -92,9 +93,9 @@ After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus NOTE: PostgreSQL and Redis should have already been disabled on the -application servers, and connections from the application servers to those -services on the backend servers configured, during normal GitLab multi-node set up. See -multi-node configuration documentation for +application servers during normal GitLab multi-node setup. Connections +from the application servers to services on the backend servers should +have also been configured. See multi-node configuration documentation for [PostgreSQL](../../postgresql/replication_and_failover.md#configuring-the-application-nodes) and [Redis](../../redis/replication_and_failover.md#example-configuration-for-the-gitlab-application). @@ -120,12 +121,12 @@ major differences: called the "tracking database", which tracks the synchronization state of various resources. -Therefore, we set up the multi-node components one-by-one, and include deviations -from the normal multi-node setup. However, we highly recommend first configuring a -brand-new cluster as if it were not part of a Geo setup so that it can be -tested and verified as a working cluster. And only then should it be modified -for use as a Geo **secondary**. This helps to separate problems that are related -and are not related to Geo setup. +Therefore, we set up the multi-node components one by one and include deviations +from the normal multi-node setup. However, we highly recommend configuring a +brand-new cluster first, as if it were not part of a Geo setup. This allows +verifying that it is a working cluster. And only then should it be modified +for use as a Geo **secondary**. This helps to separate Geo setup problems from +unrelated problems. ### Step 1: Configure the Redis and Gitaly services on the **secondary** node @@ -218,11 +219,10 @@ the **primary** database. Use the following as a guide. prometheus['enable'] = false redis['enable'] = false redis_exporter['enable'] = false - repmgr['enable'] = false + patroni['enable'] = false sidekiq['enable'] = false sidekiq_cluster['enable'] = false puma['enable'] = false - unicorn['enable'] = false ``` After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. @@ -290,11 +290,10 @@ Configure the tracking database. prometheus['enable'] = false redis['enable'] = false redis_exporter['enable'] = false - repmgr['enable'] = false + patroni['enable'] = false sidekiq['enable'] = false sidekiq_cluster['enable'] = false puma['enable'] = false - unicorn['enable'] = false ``` After making these changes, [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. @@ -366,10 +365,10 @@ then make the following modifications: ``` NOTE: -If you had set up PostgreSQL cluster using the omnibus package and you had set -up `postgresql['sql_user_password'] = 'md5 digest of secret'` setting, keep in +If you had set up PostgreSQL cluster using the omnibus package and had set +`postgresql['sql_user_password'] = 'md5 digest of secret'`, keep in mind that `gitlab_rails['db_password']` and `geo_secondary['db_password']` -mentioned above contains the plaintext passwords. This is used to let the Rails +contains the plaintext passwords. This is used to let the Rails servers connect to the databases. NOTE: @@ -438,9 +437,8 @@ application servers above, with some changes to run only the `sidekiq` service: prometheus['enable'] = false redis['enable'] = false redis_exporter['enable'] = false - repmgr['enable'] = false + patroni['enable'] = false puma['enable'] = false - unicorn['enable'] = false ## ## The unique identifier for the Geo node. diff --git a/doc/administration/geo/replication/remove_geo_node.md b/doc/administration/geo/replication/remove_geo_node.md index 697d8c6ae38..b72cd3cbb95 100644 --- a/doc/administration/geo/replication/remove_geo_node.md +++ b/doc/administration/geo/replication/remove_geo_node.md @@ -1,5 +1,6 @@ --- redirect_to: '../../geo/replication/remove_geo_site.md' +remove_date: '2021-06-01' --- This document was moved to [another location](../../geo/replication/remove_geo_site.md). diff --git a/doc/administration/geo/replication/security_review.md b/doc/administration/geo/replication/security_review.md index f84d7a2171d..ae41599311b 100644 --- a/doc/administration/geo/replication/security_review.md +++ b/doc/administration/geo/replication/security_review.md @@ -184,7 +184,7 @@ from [owasp.org](https://owasp.org/). ### What databases and application servers support the application? -- PostgreSQL >= 11, Redis, Sidekiq, Puma. +- PostgreSQL >= 12, Redis, Sidekiq, Puma. ### How will database connection strings, encryption keys, and other sensitive components be stored, accessed, and protected from unauthorized detection? diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 6d990fd12ba..1fd923dbaf1 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -583,64 +583,6 @@ to start again from scratch, there are a few steps that can help you: gitlab-ctl start ``` -## Fixing errors during a PostgreSQL upgrade or downgrade - -### Message: `ERROR: psql: FATAL: role "gitlab-consul" does not exist` - -When -[upgrading PostgreSQL on a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance), you might encounter the -following error: - -```plaintext -$ sudo gitlab-ctl pg-upgrade --target-version=11 -Checking for an omnibus managed postgresql: OK -Checking if postgresql['version'] is set: OK -Checking if we already upgraded: NOT OK -Checking for a newer version of PostgreSQL to install -Upgrading PostgreSQL to 11.7 -Checking if PostgreSQL bin files are symlinked to the expected location: OK -Waiting 30 seconds to ensure tasks complete before PostgreSQL upgrade. -See https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server for details -If you do not want to upgrade the PostgreSQL server at this time, enter Ctrl-C and see the documentation for details - -Please hit Ctrl-C now if you want to cancel the operation. -..............................Detected an HA cluster. -Error running command: /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul -ERROR: psql: FATAL: role "gitlab-consul" does not exist -Traceback (most recent call last): - 10: from /opt/gitlab/embedded/bin/omnibus-ctl:23:in `
' - 9: from /opt/gitlab/embedded/bin/omnibus-ctl:23:in `load' - 8: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/bin/omnibus-ctl:31:in `' - 7: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/lib/omnibus-ctl.rb:746:in `run' - 6: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/omnibus-ctl-0.6.0/lib/omnibus-ctl.rb:204:in `block in add_command_under_category' - 5: from /opt/gitlab/embedded/service/omnibus-ctl/pg-upgrade.rb:171:in `block in load_file' - 4: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:248:in `is_master?' - 3: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:100:in `execute_psql' - 2: from /opt/gitlab/embedded/service/omnibus-ctl-ee/lib/repmgr.rb:113:in `cmd' - 1: from /opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/mixlib-shellout-3.0.9/lib/mixlib/shellout.rb:287:in `error!' -/opt/gitlab/embedded/lib/ruby/gems/2.6.0/gems/mixlib-shellout-3.0.9/lib/mixlib/shellout.rb:300:in `invalid!': Expected process to exit with [0], but received '2' (Mixlib::ShellOut::ShellCommandFailed) ----- Begin output of /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul ---- -STDOUT: -STDERR: psql: FATAL: role "gitlab-consul" does not exist ----- End output of /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul ---- -Ran /opt/gitlab/embedded/bin/psql -qt -d gitlab_repmgr -h /var/opt/gitlab/postgresql -p 5432 -c "SELECT name FROM repmgr_gitlab_cluster.repl_nodes WHERE type='master' AND active != 'f'" -U gitlab-consul returned 2 -``` - -If you are upgrading the PostgreSQL read-replica of a Geo secondary node, and -you are not using `consul` or `repmgr`, you may need to disable `consul` and/or -`repmgr` services in `gitlab.rb`: - -```ruby -consul['enable'] = false -repmgr['enable'] = false -``` - -Then reconfigure GitLab: - -```shell -sudo gitlab-ctl reconfigure -``` - ## Fixing errors during a failover or when promoting a secondary to a primary node The following are possible errors that might be encountered during failover or @@ -756,6 +698,30 @@ this command reports `ERROR - Replication is not up-to-date` even if replication is actually up-to-date. If replication and verification output shows that it is complete, you can add `--skip-preflight-checks` to make the command complete promotion. This bug was fixed in GitLab 13.8 and later. +### Errors when using `--skip-preflight-checks` or `--force` + +Before GitLab 13.5, you could bump into one of the following errors when using +`--skip-preflight-checks` or `--force`: + +```plaintext +get_ctl_options': invalid option: --skip-preflight-checks (OptionParser::InvalidOption) + +get_ctl_options': invalid option: --force (OptionParser::InvalidOption) +``` + +This can happen with XFS or filesystems that list files in lexical order, because the +load order of the Omnibus command files can be different than expected, and a global function would get redefined. +More details can be found in [the related issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6076). + +The workaround is to manually run the preflight checks and promote the database, by running +the following commands on the Geo secondary site: + +```shell +sudo gitlab-ctl promotion-preflight-checks +sudo /opt/gitlab/embedded/bin/gitlab-pg-ctl promote +sudo gitlab-ctl reconfigure +sudo gitlab-rake geo:set_secondary_as_primary + ## Expired artifacts If you notice for some reason there are more artifacts on the Geo @@ -854,6 +820,11 @@ To resolve this issue: the **primary** node using IPv4 in the `/etc/hosts` file. Alternatively, you should [enable IPv6 on the **primary** node](https://docs.gitlab.com/omnibus/settings/nginx.html#setting-the-nginx-listen-address-or-addresses). +### Geo Admin Area shows 'Unknown' for health status and 'Request failed with status code 401' + +If using a load balancer, ensure that the load balancer's URL is set as the `external_url` in the +`/etc/gitlab/gitlab.rb` of the nodes behind the load balancer. + ### GitLab Pages return 404 errors after promoting This is due to [Pages data not being managed by Geo](datatypes.md#limitations-on-replicationverification). diff --git a/doc/administration/geo/replication/using_a_geo_server.md b/doc/administration/geo/replication/using_a_geo_server.md index f8ce72ac3f8..04c30514a89 100644 --- a/doc/administration/geo/replication/using_a_geo_server.md +++ b/doc/administration/geo/replication/using_a_geo_server.md @@ -1,5 +1,6 @@ --- redirect_to: '../../geo/replication/usage.md' +remove_date: '2022-06-01' --- This document was moved to [another location](../../geo/replication/usage.md). diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index 4a101c52325..301be931b29 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -11,6 +11,10 @@ Review this page for update instructions for your version. These steps accompany the [general steps](updating_the_geo_nodes.md#general-update-steps) for updating Geo nodes. +## Updating to GitLab 13.11 + +We found an [issue with Git clone/pull through HTTP(s)](https://gitlab.com/gitlab-org/gitlab/-/issues/330787) on Geo secondaries and on any GitLab instance if maintenance mode is enabled. This was caused by a regression in GitLab Workhorse. This is fixed in the [GitLab 13.11.4 patch release](https://about.gitlab.com/releases/2021/05/14/gitlab-13-11-4-released/). To avoid this issue, upgrade to GitLab 13.11.4 or later. + ## Updating to GitLab 13.9 We've detected an issue [with a column rename](https://gitlab.com/gitlab-org/gitlab/-/issues/324160) @@ -78,6 +82,12 @@ paused fails. Do not pause replication before promoting a secondary. If the node is paused, be sure to resume before promoting. To avoid this issue, upgrade to GitLab 13.4 or later. +WARNING: +Promoting the database during a failover can fail on XFS and filesystems ordering files lexically, +when using `--force` or `--skip-preflight-checks`, due to [an issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6076) fixed in 13.5. +The [troubleshooting steps](troubleshooting.md#errors-when-using---skip-preflight-checks-or---force) +contain a workaround if you run into errors during the failover. + ## Updating to GitLab 13.2 In GitLab 13.2, promoting a secondary node to a primary while the secondary is diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index b87a606e349..f6e72092a5f 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -9,7 +9,7 @@ type: howto NOTE: If your GitLab installation uses external (not managed by Omnibus) PostgreSQL -instances, the Omnibus roles will not be able to perform all necessary +instances, the Omnibus roles are unable to perform all necessary configuration steps. In this case, [follow the Geo with external PostgreSQL instances document instead](external_database.md). @@ -25,10 +25,23 @@ size. You are encouraged to first read through all the steps before executing them in your testing/production environment. -## PostgreSQL replication +## Single instance database replication -The GitLab **primary** node where the write operations happen will connect to -the **primary** database server, and **secondary** nodes will +A single instance database replication is easier to set up and still provides the same Geo capabilities +as a clusterized alternative. It's useful for setups running on a single machine +or trying to evaluate Geo for a future clusterized installation. + +A single instance can be expanded to a clusterized version using Patroni, which is recommended for a +highly available architecture. + +Follow below the instructions on how to set up PostgreSQL replication as a single instance database. +Alternatively, you can look at the [Multi-node database replication](#multi-node-database-replication) +instructions on setting up replication with a Patroni cluster. + +### PostgreSQL replication + +The GitLab **primary** node where the write operations happen connects to +the **primary** database server, and **secondary** nodes connect to their own database servers (which are also read-only). We recommend using [PostgreSQL replication slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75) @@ -37,8 +50,8 @@ recover. See below for more details. The following guide assumes that: -- You are using Omnibus and therefore you are using PostgreSQL 11 or later - which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/11/app-pgbasebackup.html). +- You are using Omnibus and therefore you are using PostgreSQL 12 or later + which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/12/app-pgbasebackup.html). - You have a **primary** node already set up (the GitLab server you are replicating from), running Omnibus' PostgreSQL (or equivalent version), and you have a new **secondary** server set up with the same versions of the OS, @@ -48,7 +61,7 @@ WARNING: Geo works with streaming replication. Logical replication is not supported at this time. There is an [issue where support is being discussed](https://gitlab.com/gitlab-org/gitlab/-/issues/7420). -### Step 1. Configure the **primary** server +#### Step 1. Configure the **primary** server 1. SSH into your GitLab **primary** server and login as root: @@ -75,13 +88,9 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o gitlab-ctl set-geo-primary-node ``` - This command will use your defined `external_url` in `/etc/gitlab/gitlab.rb`. - -1. GitLab 10.4 and up only: Do the following to make sure the `gitlab` database user has a password defined: + This command uses your defined `external_url` in `/etc/gitlab/gitlab.rb`. - NOTE: - Until FDW settings are removed in GitLab version 14.0, avoid using single or double quotes in the - password for PostgreSQL as that will lead to errors when reconfiguring. +1. Define a password for the `gitlab` database user: Generate a MD5 hash of the desired password: @@ -103,18 +112,28 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o # must be present in all application nodes. gitlab_rails['db_password'] = '' ``` + +1. Define a password for the database [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication). -1. Omnibus GitLab already has a [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication) - called `gitlab_replicator`. You must set the password for this user manually. - You will be prompted to enter a password: + We will use the username defined in `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` + setting. The default value is `gitlab_replicator`, but if you changed it to something else, adapt + the instructions below. + + Generate a MD5 hash of the desired password: ```shell - gitlab-ctl set-replication-password + gitlab-ctl pg-password-md5 gitlab_replicator + # Enter password: + # Confirm password: + # 950233c0dfc2f39c64cf30457c3b7f1e ``` - This command will also read the `postgresql['sql_replication_user']` Omnibus - setting in case you have changed `gitlab_replicator` username to something - else. + Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab_replicator` + postgresql['sql_replication_password'] = '' + ``` If you are using an external database not managed by Omnibus GitLab, you need to create the replicator user and define a password to it manually: @@ -154,7 +173,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o echo "External address: $(curl --silent "ipinfo.io/ip")" ``` - In most cases, the following addresses will be used to configure GitLab + In most cases, the following addresses are used to configure GitLab Geo: | Configuration | Address | @@ -168,11 +187,11 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o `postgresql['md5_auth_cidr_addresses']` and `postgresql['listen_address']`. The `listen_address` option opens PostgreSQL up to network connections with the interface - corresponding to the given address. See [the PostgreSQL documentation](https://www.postgresql.org/docs/11/runtime-config-connection.html) + corresponding to the given address. See [the PostgreSQL documentation](https://www.postgresql.org/docs/12/runtime-config-connection.html) for more details. NOTE: - If you need to use `0.0.0.0` or `*` as the listen_address, you will also need to add + If you need to use `0.0.0.0` or `*` as the listen_address, you also need to add `127.0.0.1/32` to the `postgresql['md5_auth_cidr_addresses']` setting, to allow Rails to connect through `127.0.0.1`. For more information, see [omnibus-5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). @@ -190,7 +209,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## Geo Primary role ## - configure dependent flags automatically to enable Geo ## - roles ['geo_primary_role'] + roles(['geo_primary_role']) ## ## Primary address @@ -226,7 +245,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` You may also want to edit the `wal_keep_segments` and `max_wal_senders` to match your - database replication requirements. Consult the [PostgreSQL - Replication documentation](https://www.postgresql.org/docs/11/runtime-config-replication.html) + database replication requirements. Consult the [PostgreSQL - Replication documentation](https://www.postgresql.org/docs/12/runtime-config-replication.html) for more information. 1. Save the file and reconfigure GitLab for the database listen changes and @@ -262,7 +281,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o `5432` to the **primary** server's private address. 1. A certificate was automatically generated when GitLab was reconfigured. This - will be used automatically to protect your PostgreSQL traffic from + is used automatically to protect your PostgreSQL traffic from eavesdroppers, but to protect against active ("man-in-the-middle") attackers, the **secondary** node needs a copy of the certificate. Make a copy of the PostgreSQL `server.crt` file on the **primary** node by running this command: @@ -272,10 +291,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` Copy the output into a clipboard or into a local file. You - will need it when setting up the **secondary** node! The certificate is not sensitive + need it when setting up the **secondary** node! The certificate is not sensitive data. -### Step 2. Configure the **secondary** server +#### Step 2. Configure the **secondary** server 1. SSH into your GitLab **secondary** server and login as root: @@ -325,7 +344,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o -T server.crt ~gitlab-psql/.postgresql/root.crt ``` - PostgreSQL will now only recognize that exact certificate when verifying TLS + PostgreSQL now only recognizes that exact certificate when verifying TLS connections. The certificate can only be replicated by someone with access to the private key, which is **only** present on the **primary** node. @@ -363,7 +382,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## Geo Secondary role ## - configure dependent flags automatically to enable Geo ## - roles ['geo_secondary_role'] + roles(['geo_secondary_role']) ## ## Secondary address @@ -376,12 +395,13 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## Database credentials password (defined previously in primary node) ## - replicate same values here as defined in primary node ## + postgresql['sql_replication_password'] = '' postgresql['sql_user_password'] = '' gitlab_rails['db_password'] = '' ``` For external PostgreSQL instances, see [additional instructions](external_database.md). - If you bring a former **primary** node back online to serve as a **secondary** node, then you also need to remove `roles ['geo_primary_role']` or `geo_primary_role['enable'] = true`. + If you bring a former **primary** node back online to serve as a **secondary** node, then you also need to remove `roles(['geo_primary_role'])` or `geo_primary_role['enable'] = true`. 1. Reconfigure GitLab for the changes to take effect: @@ -395,7 +415,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o gitlab-ctl restart postgresql ``` -### Step 3. Initiate the replication process +#### Step 3. Initiate the replication process Below we provide a script that connects the database on the **secondary** node to the database on the **primary** node, replicates the database, and creates the @@ -423,7 +443,7 @@ data before running `pg_basebackup`. WARNING: Each Geo **secondary** node must have its own unique replication slot name. - Using the same slot name between two secondaries will break PostgreSQL replication. + Using the same slot name between two secondaries breaks PostgreSQL replication. ```shell gitlab-ctl replicate-geo-database \ @@ -441,57 +461,57 @@ data before running `pg_basebackup`. to list them all, but here are a couple of tips: - If PostgreSQL is listening on a non-standard port, add `--port=` as well. - - If your database is too large to be transferred in 30 minutes, you will need + - If your database is too large to be transferred in 30 minutes, you need to increase the timeout, e.g., `--backup-timeout=3600` if you expect the initial replication to take under an hour. - Pass `--sslmode=disable` to skip PostgreSQL TLS authentication altogether (e.g., you know the network path is secure, or you are using a site-to-site VPN). This is **not** safe over the public Internet! - You can read more details about each `sslmode` in the - [PostgreSQL documentation](https://www.postgresql.org/docs/11/libpq-ssl.html#LIBPQ-SSL-PROTECTION); + [PostgreSQL documentation](https://www.postgresql.org/docs/12/libpq-ssl.html#LIBPQ-SSL-PROTECTION); the instructions above are carefully written to ensure protection against both passive eavesdroppers and active "man-in-the-middle" attackers. - Change the `--slot-name` to the name of the replication slot - to be used on the **primary** database. The script will attempt to create the + to be used on the **primary** database. The script attempts to create the replication slot automatically if it does not exist. - - If you're repurposing an old server into a Geo **secondary** node, you'll need to + - If you're repurposing an old server into a Geo **secondary** node, you need to add `--force` to the command line. - When not in a production machine you can disable backup step if you really sure this is what you want by adding `--skip-backup` The replication process is now complete. -## PgBouncer support (optional) +### PgBouncer support (optional) [PgBouncer](https://www.pgbouncer.org/) may be used with GitLab Geo to pool -PostgreSQL connections. We recommend using PgBouncer if you use GitLab in a -high-availability configuration with a cluster of nodes supporting a Geo -**primary** site and two other clusters of nodes supporting a Geo **secondary** site. -One for the main database and the other for the tracking database. For more information, +PostgreSQL connections, which can improve performance even when using in a +single instance installation. + +We recommend using PgBouncer if you use GitLab in a highly available +configuration with a cluster of nodes supporting a Geo **primary** site and +two other clusters of nodes supporting a Geo **secondary** site. One for the +main database and the other for the tracking database. For more information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). -## Patroni support +## Multi-node database replication -Support for Patroni is intended to replace `repmgr` as a -[highly available PostgreSQL solution](../../postgresql/replication_and_failover.md) -on the primary node, but it can also be used for PostgreSQL HA on a secondary -site. Similar to `repmgr`, using Patroni on a secondary node is optional. +In GitLab 14.0, Patroni replaced `repmgr` as the supported +[highly available PostgreSQL solution](../../postgresql/replication_and_failover.md). -Starting with GitLab 13.5, Patroni is available for _experimental_ use with Geo -primary and secondary sites. Due to its experimental nature, Patroni support is -subject to change without notice. +NOTE: +If you still haven't [migrated from repmgr to Patroni](#migrating-from-repmgr-to-patroni) you're highly advised to do so. -This experimental implementation has the following limitations: +### Patroni support -- Whenever `gitlab-ctl reconfigure` runs on a Patroni Leader instance, there's a - chance the node will be demoted due to the required short-time restart. To - avoid this, you can pause auto-failover by running `gitlab-ctl patroni pause`. - After a reconfigure, it resumes on its own. +Patroni is the official replication management solution for Geo. It +can be used to build a highly available cluster on the **primary** and a **secondary** Geo site. +Using Patroni on a **secondary** site is optional and you don't have to use the same amount of +nodes on each Geo site. For instructions about how to set up Patroni on the primary site, see the [PostgreSQL replication and failover with Omnibus GitLab](../../postgresql/replication_and_failover.md#patroni) page. -### Configuring Patroni cluster for a Geo secondary site +#### Configuring Patroni cluster for a Geo secondary site In a Geo secondary site, the main PostgreSQL database is a read-only replica of the primary site’s PostgreSQL database. @@ -503,7 +523,7 @@ configuration for the secondary site. The internal load balancer provides a sing endpoint for connecting to the Patroni cluster's leader whenever a new leader is elected. Be sure to use [password credentials](../../postgresql/replication_and_failover.md#database-authorization-for-patroni) and other database best practices. -#### Step 1. Configure Patroni permanent replication slot on the primary site +##### Step 1. Configure Patroni permanent replication slot on the primary site To set up database replication with Patroni on a secondary node, we need to configure a _permanent replication slot_ on the primary node's Patroni cluster, @@ -521,16 +541,16 @@ Leader instance**: 1. Edit `/etc/gitlab/gitlab.rb` and add the following: ```ruby - consul['enable'] = true + roles(['patroni_role']) + + consul['services'] = %w(postgresql) consul['configuration'] = { retry_join: %w[CONSUL_PRIMARY1_IP CONSUL_PRIMARY2_IP CONSUL_PRIMARY3_IP] } - - repmgr['enable'] = false - + # You need one entry for each secondary, with a unique name following PostgreSQL slot_name constraints: # - # Configuration syntax will be: 'unique_slotname' => { 'type' => 'physical' }, + # Configuration syntax is: 'unique_slotname' => { 'type' => 'physical' }, # We don't support setting a permanent replication slot for logical replication type patroni['replication_slots'] = { 'geo_secondary' => { 'type' => 'physical' } @@ -539,15 +559,18 @@ Leader instance**: patroni['use_pg_rewind'] = true patroni['postgresql']['max_wal_senders'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary). patroni['postgresql']['max_replication_slots'] = 8 # Use double of the amount of patroni/reserved slots (3 patronis + 1 reserved slot for a Geo secondary). + patroni['replication_password'] = 'PLAIN_TEXT_POSTGRESQL_REPLICATION_PASSWORD' - postgresql['md5_auth_cidr_addresses'] = [ - 'PATRONI_PRIMARY1_IP/32', 'PATRONI_PRIMARY2_IP/32', 'PATRONI_PRIMARY3_IP/32', 'PATRONI_PRIMARY_PGBOUNCER/32', - 'PATRONI_SECONDARY1_IP/32', 'PATRONI_SECONDARY2_IP/32', 'PATRONI_SECONDARY3_IP/32', 'PATRONI_SECONDARY_PGBOUNCER/32' # We list all secondary instances as they can all become a Standby Leader + # We list all secondary instances as they can all become a Standby Leader + postgresql['md5_auth_cidr_addresses'] = %w[ + PATRONI_PRIMARY1_IP/32 PATRONI_PRIMARY2_IP/32 PATRONI_PRIMARY3_IP/32 PATRONI_PRIMARY_PGBOUNCER/32 + PATRONI_SECONDARY1_IP/32 PATRONI_SECONDARY2_IP/32 PATRONI_SECONDARY3_IP/32 PATRONI_SECONDARY_PGBOUNCER/32 ] postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH' postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + postgresql['listen_address'] = '0.0.0.0' # You can use a public or VPC address here instead ``` 1. Reconfigure GitLab for the changes to take effect: @@ -556,17 +579,17 @@ Leader instance**: gitlab-ctl reconfigure ``` -#### Step 2. Configure the internal load balancer on the primary site +##### Step 2. Configure the internal load balancer on the primary site To avoid reconfiguring the Standby Leader on the secondary site whenever a new -Leader is elected on the primary site, we'll need to set up a TCP internal load -balancer which will give a single endpoint for connecting to the Patroni +Leader is elected on the primary site, we need to set up a TCP internal load +balancer which gives a single endpoint for connecting to the Patroni cluster's Leader. The Omnibus GitLab packages do not include a Load Balancer. Here's how you could do it with [HAProxy](https://www.haproxy.org/). -The following IPs and names will be used as an example: +The following IPs and names are used as an example: - `10.6.0.21`: Patroni 1 (`patroni1.internal`) - `10.6.0.21`: Patroni 2 (`patroni2.internal`) @@ -600,7 +623,7 @@ backend postgresql Refer to your preferred Load Balancer's documentation for further guidance. -#### Step 3. Configure a PgBouncer node on the secondary site +##### Step 3. Configure a PgBouncer node on the secondary site A production-ready and highly available configuration requires at least three Consul nodes, a minimum of one PgBouncer node, but it’s recommended to have @@ -621,22 +644,26 @@ Follow the minimal configuration for the PgBouncer node: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # PgBouncer configuration + pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) pgbouncer['users'] = { + 'gitlab-consul': { + # Generate it with: `gitlab-ctl pg-password-md5 gitlab-consul` + password: 'GITLAB_CONSUL_PASSWORD_HASH' + }, 'pgbouncer': { + # Generate it with: `gitlab-ctl pg-password-md5 pgbouncer` password: 'PGBOUNCER_PASSWORD_HASH' } } # Consul configuration consul['watchers'] = %w(postgresql) - consul['configuration'] = { retry_join: %w[CONSUL_SECONDARY1_IP CONSUL_SECONDARY2_IP CONSUL_SECONDARY3_IP] } - consul['monitoring_service_discovery'] = true ``` @@ -652,17 +679,17 @@ Follow the minimal configuration for the PgBouncer node: gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul ``` -1. Restart the PgBouncer service: +1. Reload the PgBouncer service: ```shell - gitlab-ctl restart pgbouncer + gitlab-ctl hup pgbouncer ``` -#### Step 4. Configure a Standby cluster on the secondary site +##### Step 4. Configure a Standby cluster on the secondary site NOTE: If you are converting a secondary site to a Patroni Cluster, you must start -on the PostgreSQL instance. It will become the Patroni Standby Leader instance, +on the PostgreSQL instance. It becomes the Patroni Standby Leader instance, and then you can switchover to another replica if you need. For each Patroni instance on the secondary site: @@ -676,21 +703,18 @@ For each Patroni instance on the secondary site: 1. Edit `/etc/gitlab/gitlab.rb` and add the following: ```ruby - roles ['consul_role', 'postgres_role'] + roles(['consul_role', 'patroni_role']) consul['enable'] = true consul['configuration'] = { retry_join: %w[CONSUL_SECONDARY1_IP CONSUL_SECONDARY2_IP CONSUL_SECONDARY3_IP] } - repmgr['enable'] = false - postgresql['md5_auth_cidr_addresses'] = [ 'PATRONI_SECONDARY1_IP/32', 'PATRONI_SECONDARY2_IP/32', 'PATRONI_SECONDARY3_IP/32', 'PATRONI_SECONDARY_PGBOUNCER/32', # Any other instance that needs access to the database as per documentation ] - patroni['enable'] = false patroni['standby_cluster']['enable'] = true patroni['standby_cluster']['host'] = 'INTERNAL_LOAD_BALANCER_PRIMARY_IP' patroni['standby_cluster']['port'] = INTERNAL_LOAD_BALANCER_PRIMARY_PORT @@ -699,6 +723,15 @@ For each Patroni instance on the secondary site: patroni['use_pg_rewind'] = true patroni['postgresql']['max_wal_senders'] = 5 # A minimum of three for one replica, plus two for each additional replica patroni['postgresql']['max_replication_slots'] = 5 # A minimum of three for one replica, plus two for each additional replica + + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' + postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH' + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + postgresql['listen_address'] = '0.0.0.0' # You can use a public or VPC address here instead + + gitlab_rails['dbpassword'] = 'POSTGRESQL_PASSWORD' + gitlab_rails['enable'] = true + gitlab_rails['auto_migrate'] = false ``` 1. Reconfigure GitLab for the changes to take effect. @@ -708,33 +741,11 @@ For each Patroni instance on the secondary site: gitlab-ctl reconfigure ``` -1. Remove the PostgreSQL data directory: - - WARNING: - If you are converting a secondary site to a Patroni Cluster, you must skip - this step on the PostgreSQL instance. - - ```shell - rm -rf /var/opt/gitlab/postgresql/data - ``` - -1. Edit `/etc/gitlab/gitlab.rb` to enable Patroni: - - ```ruby - patroni['enable'] = true - ``` - -1. Reconfigure GitLab for the changes to take effect: - - ```shell - gitlab-ctl reconfigure - ``` - ### Migrating from repmgr to Patroni 1. Before migrating, it is recommended that there is no replication lag between the primary and secondary sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. 1. Follow the [instructions to migrate repmgr to Patroni](../../postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni). When configuring Patroni on each primary site database node, add `patroni['replication_slots'] = { '' => 'physical' }` -to `gitlab.rb` where `` is the name of the replication slot for your Geo secondary. This will ensure that Patroni recognizes the replication slot as permanent and will not drop it upon restarting. +to `gitlab.rb` where `` is the name of the replication slot for your Geo secondary. This ensures that Patroni recognizes the replication slot as permanent and not drop it upon restarting. 1. If database replication to the secondary was paused before migration, resume replication once Patroni is confirmed working on the primary. ### Migrating a single PostgreSQL node to Patroni @@ -750,14 +761,14 @@ With Patroni it's now possible to support that. In order to migrate the existing 1. [Configure a Standby Cluster](#step-4-configure-a-standby-cluster-on-the-secondary-site) on that single node machine. -You will end up with a "Standby Cluster" with a single node. That allows you to later on add additional Patroni nodes +You end up with a "Standby Cluster" with a single node. That allows you to later on add additional Patroni nodes by following the same instructions above. ### Configuring Patroni cluster for the tracking PostgreSQL database Secondary sites use a separate PostgreSQL installation as a tracking database to keep track of replication status and automatically recover from potential replication issues. -Omnibus automatically configures a tracking database when `roles ['geo_secondary_role']` is set. +Omnibus automatically configures a tracking database when `roles(['geo_secondary_role'])` is set. If you want to run this database in a highly available configuration, follow the instructions below. A production-ready and secure setup requires at least three Consul nodes, three @@ -782,7 +793,7 @@ Follow the minimal configuration for the PgBouncer node for the tracking databas ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # PgBouncer configuration pgbouncer['users'] = { @@ -844,7 +855,7 @@ For each Patroni instance on the secondary site for the tracking database: ```ruby # Disable all components except PostgreSQL, Patroni, and Consul - roles ['patroni_role'] + roles(['patroni_role']) # Consul configuration consul['services'] = %w(postgresql) @@ -875,6 +886,7 @@ For each Patroni instance on the secondary site for the tracking database: # GitLab database settings gitlab_rails['db_database'] = 'gitlabhq_geo_production' gitlab_rails['db_username'] = 'gitlab_geo' + gitlab_rails['enable'] = true # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -934,8 +946,8 @@ Patroni implementation on Omnibus that do not allow us to manage two different clusters on the same machine, we recommend setting up a new Patroni cluster for the tracking database by following the same instructions above. -The secondary nodes will backfill the new tracking database, and no data -synchronization will be required. +The secondary nodes backfill the new tracking database, and no data +synchronization is required. ## Troubleshooting diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index 1b0082687e6..9e187424afa 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -24,11 +24,19 @@ developed and tested. We aim to be compatible with most external sudo -i ``` -1. Edit `/etc/gitlab/gitlab.rb` and add a **unique** ID for your node (arbitrary value): +1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - # The unique identifier for the Geo node. - gitlab_rails['geo_node_name'] = '' + ## + ## Geo Primary role + ## - configure dependent flags automatically to enable Geo + ## + roles ['geo_primary_role'] + + ## + ## The unique identifier for the Geo site. + ## + gitlab_rails['geo_node_name'] = '' ``` 1. Reconfigure the **primary** node for the change to take effect: @@ -49,7 +57,7 @@ developed and tested. We aim to be compatible with most external To set up an external database, you can either: -- Set up [streaming replication](https://www.postgresql.org/docs/11/warm-standby.html#STREAMING-REPLICATION-SLOTS) yourself (for example AWS RDS, bare metal not managed by Omnibus, etc.). +- Set up [streaming replication](https://www.postgresql.org/docs/12/warm-standby.html#STREAMING-REPLICATION-SLOTS) yourself (for example AWS RDS, bare metal not managed by Omnibus, etc.). - Perform the Omnibus configuration manually as follows. #### Leverage your cloud provider's tools to replicate the primary database @@ -64,8 +72,9 @@ cloud providers: - Amazon RDS - [Creating a Read Replica](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Create) - Azure Database for PostgreSQL - [Create and manage read replicas in Azure Database for PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal) +- Google Cloud SQL - [Creating read replicas](https://cloud.google.com/sql/docs/postgres/replication/create-replica) -Once your read-only replica is set up, you can skip to [configure you secondary application node](#configure-secondary-application-nodes-to-use-the-external-read-replica). +Once your read-only replica is set up, you can skip to [configure your secondary application node](#configure-secondary-application-nodes-to-use-the-external-read-replica). #### Manually configure the primary database for replication @@ -182,9 +191,12 @@ to grant additional roles to your tracking database user (by default, this is - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. +- Google Cloud SQL requires the [`cloudsqlsuperuser`](https://cloud.google.com/sql/docs/postgres/users#default-users) role. -If you have an external database ready to be used as the tracking database, -follow the instructions below to use it: +This is for the installation of extensions during installation and upgrades. As an alternative, +[ensure the extensions are installed manually, and read about the problems that may arise during future GitLab upgrades](../../../install/postgresql_extensions.md). + +To setup an external tracking database, follow the instructions below: NOTE: If you want to use AWS RDS as a tracking database, make sure it has access to @@ -193,8 +205,12 @@ outbound rules do not apply to RDS PostgreSQL databases. Therefore, you need to rule to the read-replica's security group allowing any TCP traffic from the tracking database on port 5432. -1. Ensure that your secondary node can communicate with your tracking database by - manually changing the `pg_hba.conf` that is associated with your tracking database. +1. Set up PostgreSQL according to the + [database requirements document](../../../install/requirements.md#database). +1. Set up a `gitlab_geo` user with a password of your choice, create the `gitlabhq_geo_production` database, and make the user an owner of the database. You can see an example of this setup in the [installation from source documentation](../../../install/installation.md#6-database). +1. If you are **not** using a cloud-managed PostgreSQL database, ensure that your secondary + node can communicate with your tracking database by manually changing the + `pg_hba.conf` that is associated with your tracking database. Remember to restart PostgreSQL afterwards for the changes to take effect: ```plaintext @@ -226,9 +242,14 @@ the tracking database on port 5432. 1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) -1. Run the tracking database migrations: +1. The reconfigure should automatically create the database. If needed, you can perform this task manually. Note that this task (whether run by itself or during reconfigure) requires the database user to be a superuser. ```shell gitlab-rake geo:db:create + ``` + +1. The reconfigure should automatically migrate the database. You can migrate the database manually if needed, for example if `gitlab_rails['auto_migrate'] = false`: + + ```shell gitlab-rake geo:db:migrate ``` diff --git a/doc/administration/git_annex.md b/doc/administration/git_annex.md index 741b2a78b85..d7fb8a37b9c 100644 --- a/doc/administration/git_annex.md +++ b/doc/administration/git_annex.md @@ -1,5 +1,6 @@ --- redirect_to: 'index.md' +remove_date: '2021-07-22' --- This document was moved to [another location](index.md). diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index c9b6fd4c510..0b22df5a115 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -82,7 +82,7 @@ The following list depicts the network architecture of Gitaly: - Gitaly addresses must be specified in such a way that they resolve correctly for **all** Gitaly clients. - Gitaly clients are: - - Puma or Unicorn. + - Puma. - Sidekiq. - GitLab Workhorse. - GitLab Shell. @@ -247,7 +247,6 @@ disable enforcement. For more information, see the documentation on configuring # node_exporter['enable'] = false # Prevent database connections during 'gitlab-ctl reconfigure' - gitlab_rails['rake_cache_clear'] = false gitlab_rails['auto_migrate'] = false # Configure the gitlab-shell API callback URL. Without this, `git push` will diff --git a/doc/administration/gitaly/faq.md b/doc/administration/gitaly/faq.md new file mode 100644 index 00000000000..98a90925d32 --- /dev/null +++ b/doc/administration/gitaly/faq.md @@ -0,0 +1,90 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +--- + +# Frequently asked questions **(FREE SELF)** + +The following are answers to frequently asked questions about Gitaly and Gitaly Cluster. + +## How does Gitaly Cluster compare to Geo? + +Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: + +- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are + not aware when Gitaly Cluster is used. +- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for + an entire instance of GitLab. Users know when they are using Geo for + [replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification), + including Git data. + +The following table outlines the major differences between Gitaly Cluster and Geo: + +| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | +|:---------------|:---------|:----------|:-------------------|:----------------------------------------------------------------------------|:-----------------------------------------|:------------------------| +| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](praefect.md#automatic-failover-and-primary-election-strategies) | [Strong](praefect.md#strong-consistency) | Data storage in Git | +| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | + +For more information, see: + +- Geo [use cases](../geo/index.md#use-cases). +- Geo [architecture](../geo/index.md#architecture). + +## Are there instructions for migrating to Gitaly Cluster? + +Yes! For more information, see [Migrate to Gitaly Cluster](praefect.md#migrate-to-gitaly-cluster). + +## What are some repository storage recommendations? + +The size of the required storage can vary between instances and depends on the set +[replication factor](praefect.md#replication-factor). You might want to include implementing +repository storage redundancy. + +For a replication factor: + +- Of `1`: NFS, Gitaly, and Gitaly Cluster have roughly the same storage requirements. +- More than `1`: The amount of required storage is `used space * replication factor`. `used space` + should include any planned future growth. + +## What are some Praefect database storage requirements? + +The requirements are relatively low because the database contains only metadata of: + +- Where repositories are located. +- Some queued work. + +It depends on the number of repositories, but a useful minimum is 5-10 GB, similar to the main +GitLab application database. + +## Can the GitLab application database and the Praefect database be on the same servers? + +Yes, however Praefect should have it's own database server when using Omnibus GitLab PostgreSQL. If +there is a failover, Praefect isn't aware and starts to fail as the database it's trying to use would +either: + +- Be unavailable. +- In read-only mode. + +A future solution may allow for Praefect and Omnibus GitLab databases on the same PostgreSQL server. +For more information, see the relevant: + +- [Omnibus GitLab issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +- [Gitaly issue](https://gitlab.com/gitlab-org/gitaly/-/issues/3398). + +## Is PgBouncer required for the Praefect database? + +No, because the number of connections Praefect makes is low. You can use the same PgBouncer instance +for both the GitLab application database and the Praefect database if you wish. + +## Are there any special considerations for Gitaly Cluster when PostgreSQL is upgraded? + +There are no special requirements. Gitaly Cluster requires PostgreSQL version 11 or later. + +## Praefect database tables are empty? + +These tables are created per the [specific configuration section](praefect.md#postgresql). + +If you find you have an empty Praefect database table, see the +[relevant troubleshooting section](index.md#relation-does-not-exist-errors). diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 3935e990590..eaf9e21780d 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -235,29 +235,6 @@ A hybrid approach can be used in these instances, where each shard is configured cluster. [Variable replication factor](https://gitlab.com/groups/gitlab-org/-/epics/3372) is planned to provide greater flexibility for extremely large GitLab instances. -### Gitaly Cluster compared to Geo - -Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: - -- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are - not aware when Gitaly Cluster is used. -- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for - an entire instance of GitLab. Users know when they are using Geo for - [replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification), - including Git data. - -The following table outlines the major differences between Gitaly Cluster and Geo: - -| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | -|:---------------|:---------|:----------|:-------------------|:----------------------------------------------------------------------------|:-----------------------------------------|:------------------------| -| Gitaly Cluster | Multiple | Single | Approximately 1 ms | [Automatic](praefect.md#automatic-failover-and-primary-election-strategies) | [Strong](praefect.md#strong-consistency) | Data storage in Git | -| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | - -For more information, see: - -- Geo [use cases](../geo/index.md#use-cases). -- Geo [architecture](../geo/index.md#architecture). - ### Architecture Praefect is a router and transaction manager for Gitaly, and a required @@ -346,7 +323,6 @@ When GitLab calls a function that has a "Rugged patch", it performs two checks: the GitLab use of "Rugged patch" code. - If the feature flag is not set, GitLab tries accessing the file system underneath the Gitaly server directly. If it can, it uses the "Rugged patch": - - If using Unicorn. - If using Puma and [thread count](../../install/requirements.md#puma-threads) is set to `1`. @@ -404,25 +380,36 @@ GitLab recommends: We welcome your feedback on this process: raise a support ticket, or [comment on the epic](https://gitlab.com/groups/gitlab-org/-/epics/4916). -## Troubleshooting Gitaly +## Troubleshooting + +Refer to the information below when troubleshooting Gitaly and Gitaly Cluster. + +Before troubleshooting, see the Gitaly and Gitaly Cluster +[frequently asked questions](faq.md). + +### Troubleshoot Gitaly + +The following sections provide possible solutions to Gitaly errors. -Check [Gitaly timeouts](../../user/admin_area/settings/gitaly_timeouts.md) when troubleshooting -Gitaly. +See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings. -### Check versions when using standalone Gitaly servers +#### Check versions when using standalone Gitaly servers When using standalone Gitaly servers, you must make sure they are the same version -as GitLab to ensure full compatibility. Check **Admin Area > Overview > Gitaly Servers** on -your GitLab instance and confirm all Gitaly servers indicate that they are up to date. +as GitLab to ensure full compatibility: -### `gitaly-debug` +1. On the top bar, select **Menu >** **{admin}** **Admin** on your GitLab instance. +1. On the left sidebar, select **Overview > Gitaly Servers**. +1. Confirm all Gitaly servers indicate that they are up to date. + +#### Use `gitaly-debug` The `gitaly-debug` command provides "production debugging" tools for Gitaly and Git performance. It is intended to help production engineers and support engineers investigate Gitaly performance problems. If you're using GitLab 11.6 or newer, this tool should be installed on -your GitLab / Gitaly server already at `/opt/gitlab/embedded/bin/gitaly-debug`. +your GitLab or Gitaly server already at `/opt/gitlab/embedded/bin/gitaly-debug`. If you're investigating an older GitLab version you can compile this tool offline and copy the executable to your server: @@ -438,19 +425,19 @@ To see the help page of `gitaly-debug` for a list of supported sub-commands, run gitaly-debug -h ``` -### Commits, pushes, and clones return a 401 +#### Commits, pushes, and clones return a 401 ```plaintext remote: GitLab: 401 Unauthorized ``` -You need to sync your `gitlab-secrets.json` file with your Gitaly clients (GitLab -app nodes). +You need to sync your `gitlab-secrets.json` file with your GitLab +application nodes. -### Client side gRPC logs +#### Client side gRPC logs Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC -client has its own log file which may contain debugging information when +client has its own log file which may contain useful information when you are seeing Gitaly errors. You can control the log level of the gRPC client with the `GRPC_LOG_LEVEL` environment variable. The default level is `WARN`. @@ -461,7 +448,7 @@ You can run a gRPC trace with: sudo GRPC_TRACE=all GRPC_VERBOSITY=DEBUG gitlab-rake gitlab:gitaly:check ``` -### Server side gRPC logs +#### Server side gRPC logs gRPC tracing can also be enabled in Gitaly itself with the `GODEBUG=http2debug` environment variable. To set this in an Omnibus GitLab install: @@ -476,7 +463,7 @@ environment variable. To set this in an Omnibus GitLab install: 1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab. -### Correlating Git processes with RPCs +#### Correlating Git processes with RPCs Sometimes you need to find out which Gitaly RPC created a particular Git process. @@ -494,7 +481,7 @@ sudo cat /proc/$PID/environ | tr '\0' '\n' | grep ^CORRELATION_ID= This method isn't reliable for `git cat-file` processes, because Gitaly internally pools and re-uses those across RPCs. -### Observing `gitaly-ruby` traffic +#### Observing `gitaly-ruby` traffic [`gitaly-ruby`](configure_gitaly.md#gitaly-ruby) is an internal implementation detail of Gitaly, so, there's not that much visibility into what goes on inside @@ -502,12 +489,13 @@ so, there's not that much visibility into what goes on inside If you have Prometheus set up to scrape your Gitaly process, you can see request rates and error codes for individual RPCs in `gitaly-ruby` by -querying `grpc_client_handled_total`. Strictly speaking, this metric does -not differentiate between `gitaly-ruby` and other RPCs. However from GitLab 11.9, -all gRPC calls made by Gitaly itself are internal calls from the main Gitaly process to one of its -`gitaly-ruby` sidecars. +querying `grpc_client_handled_total`. + +- In theory, this metric does not differentiate between `gitaly-ruby` and other RPCs. +- In practice from GitLab 11.9, all gRPC calls made by Gitaly itself are internal calls from the + main Gitaly process to one of its `gitaly-ruby` sidecars. -Assuming your `grpc_client_handled_total` counter observes only Gitaly, +Assuming your `grpc_client_handled_total` counter only observes Gitaly, the following query shows you RPCs are (most likely) internally implemented as calls to `gitaly-ruby`: @@ -515,7 +503,7 @@ implemented as calls to `gitaly-ruby`: sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 ``` -### Repository changes fail with a `401 Unauthorized` error +#### Repository changes fail with a `401 Unauthorized` error If you run Gitaly on its own server and notice these conditions: @@ -541,7 +529,7 @@ Confirm the following are all true: - When any user adds or modifies a file from the repository using the GitLab UI, it immediately fails with a red `401 Unauthorized` banner. - Creating a new project and [initializing it with a README](../../user/project/working_with_projects.md#blank-projects) - successfully creates the project, but doesn't create the README. + successfully creates the project but doesn't create the README. - When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on a Gitaly client and reproducing the error, you get `401` errors when reaching the [`/api/v4/internal/allowed`](../../development/internal_api.md) endpoint: @@ -610,7 +598,7 @@ on the Gitaly server matches the one on Gitaly client. If it doesn't match, update the secrets file on the Gitaly server to match the Gitaly client, then [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### Command line tools cannot connect to Gitaly +#### Command line tools cannot connect to Gitaly gRPC cannot reach your Gitaly server if: @@ -623,22 +611,24 @@ Verify you can reach Gitaly by using TCP: sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT] ``` -If the TCP connection fails, check your network settings and your firewall rules. -If the TCP connection succeeds, your networking and firewall rules are correct. +If the TCP connection: + +- Fails, check your network settings and your firewall rules. +- Succeeds, your networking and firewall rules are correct. -If you use proxy servers in your command line environment, such as Bash, these -can interfere with your gRPC traffic. +If you use proxy servers in your command line environment such as Bash, these can interfere with +your gRPC traffic. -If you use Bash or a compatible command line environment, run the following commands -to determine whether you have proxy servers configured: +If you use Bash or a compatible command line environment, run the following commands to determine +whether you have proxy servers configured: ```shell echo $http_proxy echo $https_proxy ``` -If either of these variables have a value, your Gitaly CLI connections may be -getting routed through a proxy which cannot connect to Gitaly. +If either of these variables have a value, your Gitaly CLI connections may be getting routed through +a proxy which cannot connect to Gitaly. To remove the proxy setting, run the following commands (depending on which variables had values): @@ -647,7 +637,7 @@ unset http_proxy unset https_proxy ``` -### Permission denied errors appearing in Gitaly or Praefect logs when accessing repositories +#### Permission denied errors appearing in Gitaly or Praefect logs when accessing repositories You might see the following in Gitaly and Praefect logs: @@ -668,9 +658,87 @@ This is a GRPC call [error response code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html). If this error occurs, even though -[the Gitaly auth tokens are correctly setup](../gitaly/praefect.md#debugging-praefect), +[the Gitaly auth tokens are set up correctly](#praefect-errors-in-logs), it's likely that the Gitaly servers are experiencing [clock drift](https://en.wikipedia.org/wiki/Clock_drift). Ensure the Gitaly clients and servers are synchronized, and use an NTP time server to keep them synchronized. + +#### Gitaly not listening on new address after reconfiguring + +When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may +continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. + +When this occurs, run `sudo gitlab-ctl restart` to resolve the issue. This should no longer be +necessary because [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved. + +#### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node + +If this error occurs even though file permissions are correct, it's likely that the Gitaly node is +experiencing [clock drift](https://en.wikipedia.org/wiki/Clock_drift). + +Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time +server to keep them synchronized if possible. + +### Troubleshoot Praefect (Gitaly Cluster) + +The following sections provide possible solutions to Gitaly Cluster errors. + +#### Praefect errors in logs + +If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. + +Here are common errors and potential causes: + +- 500 response code + - **ActionView::Template::Error (7:permission denied)** + - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. + - **Unable to save project. Error: 7:permission denied** + - Secret token in `praefect['storage_nodes']` on GitLab server does not match the + value in `gitaly['auth_token']` on one or more Gitaly servers. +- 503 response code + - **GRPC::Unavailable (14:failed to connect to all addresses)** + - GitLab was unable to reach Praefect. + - **GRPC::Unavailable (14:all SubCons are in TransientFailure...)** + - Praefect cannot reach one or more of its child Gitaly nodes. Try running + the Praefect connection checker to diagnose. + +#### Determine primary Gitaly node + +To determine the current primary Gitaly node for a specific Praefect node: + +- Use the `Shard Primary Election` [Grafana chart](praefect.md#grafana) on the + [`Gitlab Omnibus - Praefect` dashboard](https://gitlab.com/gitlab-org/grafana-dashboards/-/blob/master/omnibus/praefect.json). + This is recommended. +- If you do not have Grafana set up, use the following command on each host of each + Praefect node: + + ```shell + curl localhost:9652/metrics | grep gitaly_praefect_primaries` + ``` + +#### Relation does not exist errors + +By default Praefect database tables are created automatically by `gitlab-ctl reconfigure` task. +However, if the `gitlab-ctl reconfigure` command isn't executed or there are errors during the +execution, the Praefect database tables are not created on initial reconfigure and can throw +errors that relations do not exist. + +For example: + +- `ERROR: relation "node_status" does not exist at character 13` +- `ERROR: relation "replication_queue_lock" does not exist at character 40` +- This error: + + ```json + {"level":"error","msg":"Error updating node: pq: relation \"node_status\" does not exist","pid":210882,"praefectName":"gitlab1x4m:0.0.0.0:2305","time":"2021-04-01T19:26:19.473Z","virtual_storage":"praefect-cluster-1"} + ``` + +To solve this, the database schema migration can be done using `sql-migrate` sub-command of +the `praefect` command: + +```shell +$ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate +praefect sql-migrate: OK (applied 21 migrations) +``` diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index a1733d9d6ac..21e5360e27b 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -9,13 +9,21 @@ type: reference Configure Gitaly Cluster using either: -- The Gitaly Cluster configuration instructions available as part of - [reference architectures](../reference_architectures/index.md) for installations for more than - 2000 users. -- The advanced configuration instructions that follow on this page. +- Gitaly Cluster configuration instructions available as part of + [reference architectures](../reference_architectures/index.md) for installations of up to: + - [3000 users](../reference_architectures/3k_users.md#configure-gitaly-cluster). + - [5000 users](../reference_architectures/5k_users.md#configure-gitaly-cluster). + - [10,000 users](../reference_architectures/10k_users.md#configure-gitaly-cluster). + - [25,000 users](../reference_architectures/25k_users.md#configure-gitaly-cluster). + - [50,000 users](../reference_architectures/50k_users.md#configure-gitaly-cluster). +- The custom configuration instructions that follow on this page. Smaller GitLab installations may need only [Gitaly itself](index.md). +NOTE: +Upgrade instructions for Omnibus GitLab installations +[are available](https://docs.gitlab.com/omnibus/update/#gitaly-cluster). + ## Requirements for configuring a Gitaly Cluster The minimum recommended configuration for a Gitaly Cluster requires: @@ -161,6 +169,9 @@ node, using `psql` which is installed by Omnibus GitLab. The database used by Praefect is now configured. +If you see Praefect database errors after configuring PostgreSQL, see +[troubleshooting steps](index.md#relation-does-not-exist-errors). + #### PgBouncer To reduce PostgreSQL resource consumption, we recommend setting up and configuring @@ -223,8 +234,10 @@ PostgreSQL instances. Otherwise you should change the configuration parameter > [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/2634) in GitLab 13.4, Praefect nodes can no longer be designated as `primary`. -NOTE: -If there are multiple Praefect nodes, complete these steps for **each** node. +If there are multiple Praefect nodes: + +- Complete the following steps for **each** node. +- Designate one node as the "deploy node", and configure it first. To complete this section you need a [configured PostgreSQL server](#postgresql), including: @@ -259,8 +272,8 @@ application server, or a Gitaly node. praefect['enable'] = true # Prevent database connections during 'gitlab-ctl reconfigure' - gitlab_rails['rake_cache_clear'] = false gitlab_rails['auto_migrate'] = false + praefect['auto_migrate'] = false ``` 1. Configure **Praefect** to listen on network interfaces by editing @@ -374,6 +387,30 @@ application server, or a Gitaly node. 1. [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2013) in GitLab 13.1 and later, enable [distribution of reads](#distributed-reads). +1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure + Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): + + ```shell + gitlab-ctl reconfigure + ``` + +1. For: + + - The "deploy node": + 1. Enable Praefect auto-migration again by setting `praefect['auto_migrate'] = true` in + `/etc/gitlab/gitlab.rb`. + 1. To ensure database migrations are only run during reconfigure and not automatically on + upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + - The other nodes, you can leave the settings as they are. Though + `/etc/gitlab/skip-auto-reconfigure` isn't required, you may want to set it to prevent GitLab + running reconfigure automatically when running commands such as `apt-get update`. This way any + additional configuration changes can be done and then reconfigure can be run manually. + 1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): @@ -599,7 +636,6 @@ documentation](configure_gitaly.md#configure-gitaly-servers). prometheus['enable'] = true # Prevent database connections during 'gitlab-ctl reconfigure' - gitlab_rails['rake_cache_clear'] = false gitlab_rails['auto_migrate'] = false ``` @@ -696,7 +732,7 @@ documentation](configure_gitaly.md#configure-gitaly-servers). **The steps above must be completed for each Gitaly node!** -After all Gitaly nodes are configured, you can run the Praefect connection +After all Gitaly nodes are configured, run the Praefect connection checker to verify Praefect can connect to all Gitaly servers in the Praefect configuration. @@ -865,9 +901,13 @@ Particular attention should be shown to: gitlab-rake gitlab:gitaly:check ``` -1. Check in **Admin Area > Settings > Repository > Repository storage** that the Praefect storage - is configured to store new repositories. Following this guide, the `default` storage should have - weight 100 to store all new repositories. +1. Check that the Praefect storage is configured to store new repositories: + + 1. On the top bar, select **Menu >** **{admin}** **Admin**. + 1. On the left sidebar, select **Settings > Repository**. + 1. Expand the **Repository storage** section. + + Following this guide, the `default` storage should have weight 100 to store all new repositories. 1. Verify everything is working by creating a new project. Check the "Initialize repository with a README" box so that there is content in the @@ -878,7 +918,7 @@ Particular attention should be shown to: When adding Gitaly Cluster to an existing Gitaly instance, the existing Gitaly storage must use a TCP address. If `gitaly_address` is not specified, then a Unix socket is used, -which will prevent the communication with the cluster. +which prevents the communication with the cluster. For example: @@ -1160,15 +1200,36 @@ To migrate existing clusters: a Praefect node to reattempt it. The migration only runs with `sql` election strategy configured. 1. Running two different election strategies side by side can cause a split brain, where different - Praefect nodes consider repositories to have different primaries. To avoid this, shut down - all Praefect nodes before changing the election strategy. + Praefect nodes consider repositories to have different primaries. This can be avoided either: + + - If a short downtime is acceptable: - Do this by running `gitlab-ctl stop praefect` on the Praefect nodes. + 1. Shut down all Praefect nodes before changing the election strategy. Do this by running `gitlab-ctl stop praefect` on the Praefect nodes. -1. On the Praefect nodes, configure the election strategy in `/etc/gitlab/gitlab.rb` with - `praefect['failover_election_strategy'] = 'per_repository'`. + 1. On the Praefect nodes, configure the election strategy in `/etc/gitlab/gitlab.rb` with `praefect['failover_election_strategy'] = 'per_repository'`. -1. Finally, run `gitlab-ctl reconfigure` to reconfigure and restart the Praefect nodes. + 1. Run `gitlab-ctl reconfigure && gitlab-ctl start` to reconfigure and start the Praefects. + + - If downtime is unacceptable: + + 1. Determine which Gitaly node is [the current primary](index.md#determine-primary-gitaly-node). + + 1. Comment out the secondary Gitaly nodes from the virtual storage's configuration in `/etc/gitlab/gitlab.rb` + on all Praefect nodes. This ensures there's only one Gitaly node configured, causing both of the election + strategies to elect the same Gitaly node as the primary. + + 1. Run `gitlab-ctl reconfigure` on all Praefect nodes. Wait until all Praefect processes have restarted and + the old processes have exited. This can take up to one minute. + + 1. On all Praefect nodes, configure the election strategy in `/etc/gitlab/gitlab.rb` with + `praefect['failover_election_strategy'] = 'per_repository'`. + + 1. Run `gitlab-ctl reconfigure` on all Praefect nodes. Wait until all of the Praefect processes have restarted and + the old processes have exited. This can take up to one minute. + + 1. Uncomment the secondary Gitaly node configuration commented out in the earlier step on all Praefect nodes. + + 1. Run `gitlab-ctl reconfigure` on all Praefect nodes to reconfigure and restart the Praefect processes. ### Deprecated election strategies @@ -1428,15 +1489,8 @@ or to move from single Gitaly nodes, the basic process involves: 1. Create and configure Gitaly Cluster. 1. [Move the repositories](#move-repositories). -The size of the required storage can vary between instances and depends on the set -[replication factor](#replication-factor). The migration to Gitaly Cluster might include -implementing repository storage redundancy. - -For a replication factor: - -- Of `1`: NFS, Gitaly, and Gitaly Cluster have roughly the same storage requirements. -- More than `1`: The amount of required storage is `used space * replication factor`. `used space` - should include any planned future growth. +When creating the storage, see some +[repository storage recommendations](faq.md#what-are-some-repository-storage-recommendations). ### Move Repositories @@ -1467,8 +1521,10 @@ After creating and configuring Gitaly Cluster: 1. [Schedule repository storage moves for all projects on a storage shard](../../api/project_repository_storage_moves.md#schedule-repository-storage-moves-for-all-projects-on-a-storage-shard) using the API. For example: ```shell - curl --request POST --header "Private-Token: " --header "Content-Type: application/json" \ - --data '{"source_storage_name":"","destination_storage_name":""}' "https://gitlab.example.com/api/v4/project_repository_storage_moves" + curl --request POST --header "Private-Token: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"","destination_storage_name":""}' \ + "https://gitlab.example.com/api/v4/project_repository_storage_moves" ``` 1. [Query the most recent repository moves](../../api/project_repository_storage_moves.md#retrieve-all-project-repository-storage-moves) @@ -1500,8 +1556,10 @@ After creating and configuring Gitaly Cluster: 1. [Schedule repository storage moves for all snippets on a storage shard](../../api/snippet_repository_storage_moves.md#schedule-repository-storage-moves-for-all-snippets-on-a-storage-shard) using the API. For example: ```shell - curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ - --data '{"source_storage_name":"","destination_storage_name":""}' "https://gitlab.example.com/api/v4/snippet_repository_storage_moves" + curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"","destination_storage_name":""}' \ + "https://gitlab.example.com/api/v4/snippet_repository_storage_moves" ``` 1. [Query the most recent repository moves](../../api/snippet_repository_storage_moves.md#retrieve-all-snippet-repository-storage-moves) @@ -1525,8 +1583,10 @@ After creating and configuring Gitaly Cluster: 1. [Schedule repository storage moves for all groups on a storage shard](../../api/group_repository_storage_moves.md#schedule-repository-storage-moves-for-all-groups-on-a-storage-shard) using the API. ```shell - curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ - --data '{"source_storage_name":"","destination_storage_name":""}' "https://gitlab.example.com/api/v4/group_repository_storage_moves" + curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"","destination_storage_name":""}' \ + "https://gitlab.example.com/api/v4/group_repository_storage_moves" ``` 1. [Query the most recent repository moves](../../api/group_repository_storage_moves.md#retrieve-all-group-repository-storage-moves) @@ -1544,35 +1604,3 @@ After creating and configuring Gitaly Cluster: ``` 1. Repeat for each storage as required. - -## Debugging Praefect - -If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. - -Here are common errors and potential causes: - -- 500 response code - - **ActionView::Template::Error (7:permission denied)** - - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - - **Unable to save project. Error: 7:permission denied** - - Secret token in `praefect['storage_nodes']` on GitLab server does not match the - value in `gitaly['auth_token']` on one or more Gitaly servers. -- 503 response code - - **GRPC::Unavailable (14:failed to connect to all addresses)** - - GitLab was unable to reach Praefect. - - **GRPC::Unavailable (14:all SubCons are in TransientFailure...)** - - Praefect cannot reach one or more of its child Gitaly nodes. Try running - the Praefect connection checker to diagnose. - -### Determine primary Gitaly node - -To determine the current primary Gitaly node for a specific Praefect node: - -- Use the `Shard Primary Election` [Grafana chart](#grafana) on the [`Gitlab Omnibus - Praefect` dashboard](https://gitlab.com/gitlab-org/grafana-dashboards/-/blob/master/omnibus/praefect.json). - This is recommended. -- If you do not have Grafana set up, use the following command on each host of each - Praefect node: - - ```shell - curl localhost:9652/metrics | grep gitaly_praefect_primaries` - ``` diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 9aa6bffdb98..56af5f56cfa 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -21,9 +21,8 @@ GitLab has several features based on receiving incoming emails: ## Requirements -It is **not** recommended to use an email address that receives any -messages not intended for the GitLab instance. Any incoming emails not intended -for GitLab receive a reject notice. +We recommend using an email address that receives **only** messages that are intended for +the GitLab instance. Any incoming emails not intended for GitLab receive a reject notice. Handling incoming emails requires an [IMAP](https://en.wikipedia.org/wiki/Internet_Message_Access_Protocol)-enabled email account. GitLab requires one of the following three strategies: @@ -131,6 +130,9 @@ list. ```shell sudo gitlab-ctl reconfigure + + # Needed when enabling or disabling for the first time but not for password changes. + # See https://gitlab.com/gitlab-org/gitlab-foss/-/issues/23560#note_61966788 sudo gitlab-ctl restart ``` @@ -469,11 +471,6 @@ This series of PowerShell commands enables [sub-addressing](#email-sub-addressin at the organization level in Office 365. This allows all mailboxes in the organization to receive sub-addressed mail: -NOTE: -This series of commands enables sub-addressing at the organization -level in Office 365. This allows all mailboxes in the organization -to receive sub-addressed mail. - ```powershell Set-ExecutionPolicy RemoteSigned -Scope CurrentUser diff --git a/doc/administration/index.md b/doc/administration/index.md index 1bc2084a23a..69e8689c589 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -54,7 +54,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Environment variables](environment_variables.md): Supported environment variables that can be used to override their default values to configure GitLab. -- [Plugins](file_hooks.md): With custom plugins, GitLab administrators can +- [File hooks](file_hooks.md): With custom file hooks, GitLab administrators can introduce custom integrations without modifying GitLab source code. - [Enforcing Terms of Service](../user/admin_area/settings/terms.md) - [Third party offers](../user/admin_area/settings/third_party_offers.md) @@ -103,7 +103,6 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [GitLab in maintenance mode](maintenance_mode/index.md): Put GitLab in maintenance mode. - [Update GitLab](../update/index.md): Update guides to upgrade your installation to a new version. - [Upgrading without downtime](../update/index.md#upgrading-without-downtime): Upgrade to a newer major, minor, or patch version of GitLab without taking your GitLab instance offline. -- [Migrate your GitLab CI/CD data to another version of GitLab](../migrate_ci_to_ce/README.md): If you have an old GitLab installation (older than 8.0), follow this guide to migrate your existing GitLab CI/CD data to another version of GitLab. ### Upgrading or downgrading GitLab @@ -172,7 +171,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [External Pipeline Validation](external_pipeline_validation.md): Enable, disable, and configure external pipeline validation. - [Job artifacts](job_artifacts.md): Enable, disable, and configure job artifacts (a set of files and directories which are outputted by a job when it completes successfully). - [Job logs](job_logs.md): Information about the job logs. -- [Register runners](../ci/runners/README.md#types-of-runners): Learn how to register and configure runners. +- [Register runners](../ci/runners/runners_scope.md): Learn how to register and configure runners. - [Shared runners pipelines quota](../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota): Limit the usage of pipeline minutes for shared runners. - [Enable/disable Auto DevOps](../topics/autodevops/index.md#enable-or-disable-auto-devops): Enable or disable Auto DevOps for your instance. diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 9ea76ff6151..9423045e3b5 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab application limits +# GitLab application limits **(FREE SELF)** GitLab, like most large applications, enforces limits within certain features to maintain a minimum quality of performance. Allowing some features to be limitless could affect security, @@ -112,7 +112,7 @@ Limit the maximum daily member invitations allowed per group hierarchy. - GitLab.com: Free members may invite 20 members per day. - Self-managed: Invites are not limited. -### Webhook calls +### Webhook rate limit > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61151) in GitLab 13.12. > - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. @@ -169,8 +169,8 @@ Read more about [Gitaly concurrency limits](gitaly/configure_gitaly.md#limit-rpc There's a limit to the number of comments that can be submitted on an issue, merge request, or commit. When the limit is reached, system notes can still be -added so that the history of events is not lost, but user-submitted comments -will fail. +added so that the history of events is not lost, but the user-submitted +comment fails. - **Max limit**: 5,000 comments. @@ -179,10 +179,10 @@ will fail. > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/61974) in GitLab 12.2. There is a limit to the size of comments and descriptions of issues, merge requests, and epics. -Attempting to add a body of text larger than the limit will result in an error, and the -item will not be created. +Attempting to add a body of text larger than the limit, results in an error, and the +item is also not created. -It's possible that this limit will be changed to a lower number in the future. +It's possible that this limit changes to a lower number in the future. - **Max size**: ~1 million characters / ~1 MB. @@ -191,8 +191,8 @@ It's possible that this limit will be changed to a lower number in the future. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292039) in GitLab 13.9 Commits with arbitrarily large messages may be pushed to GitLab, but when -displaying commits, titles (the first line of the commit message) will be -limited to 1KiB, and descriptions (the rest of the message) will be limited to +displaying commits, titles (the first line of the commit message) +limits to 1KiB, and descriptions (the rest of the message) limits to 1MiB. ## Number of issues in the milestone overview @@ -316,7 +316,7 @@ each time a new pipeline is created. An active pipeline is any pipeline in one o - `running` If a new pipeline would cause the total number of jobs to exceed the limit, the pipeline -will fail with a `job_activity_limit_exceeded` error. +fails with a `job_activity_limit_exceeded` error. - GitLab SaaS subscribers have different limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect all projects under that plan. @@ -367,7 +367,7 @@ The total number of subscriptions can be limited per project. This limit is checked each time a new subscription is created. If a new subscription would cause the total number of subscription to exceed the -limit, the subscription will be considered invalid. +limit, the subscription is considered invalid. - GitLab SaaS subscribers have different limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect all projects under that plan. @@ -391,7 +391,7 @@ Set the limit to `0` to disable it. The total number of pipeline schedules can be limited per project. This limit is checked each time a new pipeline schedule is created. If a new pipeline schedule would cause the total number of pipeline schedules to exceed the limit, the -pipeline schedule will not be created. +pipeline schedule is not created. GitLab SaaS subscribers have different limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect all projects under that plan. @@ -413,7 +413,7 @@ Plan.default.actual_limits.update!(ci_pipeline_schedules: 100) The total number of instance level CI/CD variables is limited at the instance level. This limit is checked each time a new instance level variable is created. If a new variable -would cause the total number of variables to exceed the limit, the new variable will not be created. +would cause the total number of variables to exceed the limit, the new variable is created. On self-managed instances this limit is defined for the `default` plan. By default, this limit is set to `25`. @@ -482,10 +482,9 @@ Plan.default.actual_limits.update!(ci_max_artifact_size_junit: 10) > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/321368) in GitLab 13.12. -The total number of registered runners is limited at the group and project -levels. Each time a new runner is registered, GitLab checks these limits. A -runner's registration fails if it exceeds the limit for the scope determined by -the runner registration token. +The total number of registered runners is limited at the group and project levels. Each time a new runner is registered, +GitLab checks these limits against runners that have been active in the last 3 months. +A runner's registration fails if it exceeds the limit for the scope determined by the runner registration token. - GitLab SaaS subscribers have different limits defined per plan, affecting all projects using that plan. - Self-managed GitLab Premium and Ultimate limits are defined by a default plan that affects all projects: @@ -574,7 +573,26 @@ See [Environment Dashboard](../ci/environments/environments_dashboard.md#adding- Pods and Deployments. However, data over 10 MB for a certain environment read from Kubernetes won't be shown. -## Merge request reports +## Merge requests + +### Diff limits + +GitLab has limits around: + +- The patch size for a single file. [This is configurable on self-managed instance](../user/admin_area/diff_limits.md). +- The total size of all the diffs for a merge request. + +An upper and lower limit applies to each of these: + +- The number of changed files. +- The number of changed lines. +- The cumulative size of the changes displayed. + +The lower limits result in additional diffs being collapsed. The higher limits +prevent any more changes from rendering. For more information about these limits, +[read the development documentation](../development/diffs.md#diff-limits). + +### Merge request reports size limit Reports that go over the 20 MB limit won't be loaded. Affected reports: @@ -589,8 +607,8 @@ Reports that go over the 20 MB limit won't be loaded. Affected reports: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8638) in GitLab 13.3. You can set a limit on the content of repository files that are indexed in -Elasticsearch. Any files larger than this limit will not be indexed, and thus -will not be searchable. +Elasticsearch. Any files larger than this limit is neither indexed +nor searchable. Setting a limit helps reduce the memory usage of the indexing processes and the overall index size. This value defaults to `1024 KiB` (1 MiB) as any @@ -598,8 +616,8 @@ text files larger than this likely aren't meant to be read by humans. You must set a limit, as unlimited file sizes aren't supported. Setting this value to be greater than the amount of memory on GitLab Sidekiq nodes causes -the GitLab Sidekiq nodes to run out of memory, as they will pre-allocate this -amount of memory during indexing. +the GitLab Sidekiq nodes to run out of memory, as this amount of memory +is pre-allocated during indexing. ### Maximum field length @@ -607,18 +625,17 @@ amount of memory during indexing. You can set a limit on the content of text fields indexed for Advanced Search. Setting a maximum helps to reduce the load of the indexing processes. If any -text field exceeds this limit then the text will be truncated to this number of -characters and the rest will not be indexed and hence will not be searchable. -This is applicable to all indexed data except repository files that get -indexed, which have a separate limit (see [Maximum file size -indexed](#maximum-file-size-indexed)). - -- On GitLab.com, this is limited to 20,000 characters -- For self-managed installations, this is unlimited by default +text field exceeds this limit, then the text is truncated to this number of +characters. The rest of the text is not indexed, and not searchable. +This applies to all indexed data except repository files that get +indexed, which have a separate limit. For more information, read +[Maximum file size indexed](#maximum-file-size-indexed). -This limit can be configured for self-managed installations when [enabling -Elasticsearch](../integration/elasticsearch.md#enabling-advanced-search). +- On GitLab.com, the field length limit is 20,000 characters. +- For self-managed installations, the field length is unlimited by default. +You can configure this limit for self-managed installations when you +[enable Elasticsearch](../integration/elasticsearch.md#enabling-advanced-search). Set the limit to `0` to disable it. ## Wiki limits @@ -653,7 +670,7 @@ More information can be found in these docs: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31007) in GitLab 12.4. Total number of changes (branches or tags) in a single push to determine whether -individual push events or bulk push event will be created. +individual push events or a bulk push event are created. More information can be found in the [Push event activities limit and bulk push events documentation](../user/admin_area/settings/push_event_activities_limit.md). diff --git a/doc/administration/integration/kroki.md b/doc/administration/integration/kroki.md index 9e9ea62c44e..e36b8a0be9d 100644 --- a/doc/administration/integration/kroki.md +++ b/doc/administration/integration/kroki.md @@ -6,10 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Kroki diagrams **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241744) in GitLab 13.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241744) in GitLab 13.7. +> - Support for reStructuredText and Textile documents [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324766) in GitLab 13.12. When [Kroki](https://kroki.io) integration is enabled and configured in -GitLab you can use it to create diagrams in AsciiDoc and Markdown documents. +GitLab you can use it to create diagrams in AsciiDoc, Markdown, reStructuredText, and Textile documents. ## Kroki Server @@ -55,7 +56,7 @@ read the [Kroki installation](https://docs.kroki.io/kroki/setup/install/#_images You need to enable Kroki integration from Settings under Admin Area. To do that, log in with an administrator account and follow these steps: -1. Select the Admin Area (**{admin}**) icon. +1. On the top bar, select **Menu >** **{admin}** **Admin**. 1. Go to **Settings > General**. 1. Expand the **Kroki** section. 1. Select **Enable Kroki** checkbox. @@ -85,13 +86,29 @@ your AsciiDoc or Markdown documentation using delimited blocks: .... ``` +- **reStructuredText** + + ```plaintext + .. code-block:: plantuml + + Bob->Alice : hello + Alice -> Bob : hi + ``` + +- **Textile** + + ```plaintext + bc[plantuml]. Bob->Alice : hello + Alice -> Bob : hi + ``` + The above blocks are converted to an HTML image tag with source pointing to the Kroki instance. If the Kroki server is correctly configured, this should render a nice diagram instead of the block: ![PlantUML diagram](../img/kroki_plantuml_diagram.png) -Kroki supports more than a dozen diagram libraries. Here's a few examples: +Kroki supports more than a dozen diagram libraries. Here's a few examples for AsciiDoc: **GraphViz** diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 834f4047fdd..2b18efde95d 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -206,7 +206,7 @@ stop; After configuring your local PlantUML server, you're ready to enable the PlantUML integration: 1. Sign in to GitLab as an [Administrator](../../user/permissions.md) user. -1. In the top menu, click **{admin}** **Admin Area**. +1. On the top bar, select **Menu >** **{admin}** **Admin**. 1. In the left sidebar, go to **Settings > General** and expand the **PlantUML** section. 1. Select the **Enable PlantUML** check box. 1. Set the PlantUML instance as `https://gitlab.example.com/-/plantuml/`, diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index 644e2d905ae..9302e9a1edc 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Web terminals **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7690) in GitLab 8.15. - With the introduction of the [Kubernetes integration](../../user/project/clusters/index.md), GitLab can store and use credentials for a Kubernetes cluster. GitLab uses these credentials to provide access to @@ -99,9 +97,10 @@ they receive a `Connection failed` message. ## Limiting WebSocket connection time -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8413) in GitLab 8.17. +By default, terminal sessions do not expire. To limit the terminal session +lifetime in your GitLab instance: -Terminal sessions, by default, do not expire. -You can limit terminal session lifetime in your GitLab instance. To do so, -go to [**Admin Area > Settings > Web terminal**](../../user/admin_area/settings/index.md#general), -and set a `max session time`. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. Select + [**Settings > Web terminal**](../../user/admin_area/settings/index.md#general). +1. Set a `max session time`. diff --git a/doc/administration/issue_closing_pattern.md b/doc/administration/issue_closing_pattern.md index 3b9ec74e6ee..31e9944d9a4 100644 --- a/doc/administration/issue_closing_pattern.md +++ b/doc/administration/issue_closing_pattern.md @@ -20,7 +20,7 @@ in the project's default branch. In order to change the pattern you need to have access to the server that GitLab is installed on. -The default pattern can be located in [`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/blob/master/config/gitlab.yml.example) +The default pattern can be located in [`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example) under the "Automatic issue closing" section. NOTE: diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 187e98e9b43..99eb1395503 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Testing info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference, howto --- @@ -113,7 +113,7 @@ For source installations the following settings are nested under `artifacts:` an | Setting | Default | Description | |---------------------|---------|-------------| | `enabled` | `false` | Enable or disable object storage. | -| `remote_directory` | | The bucket name where Artifacts are stored. | +| `remote_directory` | | The bucket name where Artifacts are stored. Use the name only, do not include the path. | | `direct_upload` | `false` | Set to `true` to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | | `background_upload` | `true` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3. | | `proxy_download` | `false` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 93f83311cad..510da68442c 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -10,7 +10,7 @@ type: reference > [Renamed from job traces to job logs](https://gitlab.com/gitlab-org/gitlab/-/issues/29121) in GitLab 12.5. Job logs are sent by a runner while it's processing a job. You can see -logs in job pages, pipelines, email notifications, etc. +logs in job pages, pipelines, email notifications, and so on. ## Data flow @@ -20,9 +20,8 @@ In the following table you can see the phases a log goes through: | Phase | State | Condition | Data flow | Stored path | | -------------- | ------------ | ----------------------- | -----------------------------------------| ----------- | | 1: patching | log | When a job is running | Runner => Puma => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | -| 2: overwriting | log | When a job is finished | Runner => Puma => file storage | `#{ROOT_PATH}/gitlab-ci/builds/#{YYYY_mm}/#{project_id}/#{job_id}.log` | -| 3: archiving | archived log | After a job is finished | Sidekiq moves log to artifacts folder | `#{ROOT_PATH}/gitlab-rails/shared/artifacts/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | -| 4: uploading | archived log | After a log is archived | Sidekiq moves archived log to [object storage](#uploading-logs-to-object-storage) (if configured) | `#{bucket_name}/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | +| 2: archiving | archived log | After a job is finished | Sidekiq moves log to artifacts folder | `#{ROOT_PATH}/gitlab-rails/shared/artifacts/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | +| 3: uploading | archived log | After a log is archived | Sidekiq moves archived log to [object storage](#uploading-logs-to-object-storage) (if configured) | `#{bucket_name}/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | The `ROOT_PATH` varies per environment. For Omnibus GitLab it would be `/var/opt/gitlab`, and for installations from source @@ -128,6 +127,11 @@ This command permanently deletes the log files and is irreversible. find /var/opt/gitlab/gitlab-rails/shared/artifacts -name "job.log" -mtime +60 -delete ``` +NOTE: +After execution, broken file references can be reported when running +[`sudo gitlab-rake gitlab:artifacts:check`](raketasks/check.md#uploaded-files-integrity). +For more information, see [delete references to missing artifacts](raketasks/check.md#delete-references-to-missing-artifacts). + ## Incremental logging architecture > - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index 552dc7095e2..b62e2ed4313 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Using the Libravatar service with GitLab +# Using the Libravatar service with GitLab **(FREE SELF)** GitLab by default supports the [Gravatar](https://gravatar.com) avatar service. @@ -17,7 +17,7 @@ server. ## Configuration -In the [`gitlab.yml` gravatar section](https://gitlab.com/gitlab-org/gitlab/blob/672bd3902d86b78d730cea809fce312ec49d39d7/config/gitlab.yml.example#L122), set +In the [`gitlab.yml` gravatar section](https://gitlab.com/gitlab-org/gitlab/-/blob/672bd3902d86b78d730cea809fce312ec49d39d7/config/gitlab.yml.example#L122), set the configuration options as follows: ### For HTTP diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md index ae96989a188..ca66791166e 100644 --- a/doc/administration/load_balancer.md +++ b/doc/administration/load_balancer.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Load Balancer for multi-node GitLab +# Load Balancer for multi-node GitLab **(FREE SELF)** In an multi-node GitLab configuration, you need a load balancer to route traffic to the application servers. The specifics on which load balancer to use diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 289e2cb5362..cf9c2143d8c 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -21,6 +21,42 @@ including adjusting log retention, log forwarding, switching logs from JSON to plain text logging, and more. - [How to parse and analyze JSON logs](troubleshooting/log_parsing.md). +## Log Rotation + +The logs for a given service may be managed and rotated by: + +- `logrotate` +- `svlogd` (`runit`'s service logging daemon) +- `logrotate` and `svlogd` +- Or not at all + +The table below includes information about what is responsible for managing and rotating logs for +the included services. Logs +[managed by `svlogd`](https://docs.gitlab.com/omnibus/settings/logs.html#runit-logs) +are written to a file called `current`. The `logrotate` service built into GitLab +[manages all logs](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate) +except those captured by `runit`. + +| Log Type | Managed by logrotate | Managed by svlogd/runit | +| ----------------------------------------------- | -------------------- | ----------------------- | +| [Alertmanager Logs](#alertmanager-logs) | N | Y | +| [Crond Logs](#crond-logs) | N | Y | +| [Gitaly](#gitaly-logs) | Y | Y | +| [GitLab Exporter For Omnibus](#gitlab-exporter) | N | Y | +| [GitLab Pages Logs](#pages-logs) | Y | Y | +| GitLab Rails | Y | N | +| [GitLab Shell Logs](#gitlab-shelllog) | Y | N | +| [Grafana Logs](#grafana-logs) | N | Y | +| [LogRotate Logs](#logrotate-logs) | N | Y | +| [Mailroom](#mail_room_jsonlog-default) | Y | Y | +| [NGINX](#nginx-logs) | Y | Y | +| [PostgreSQL Logs](#postgresql-logs) | N | Y | +| [Prometheus Logs](#prometheus-logs) | N | Y | +| [Puma](#puma-logs) | Y | Y | +| [Redis Logs](#redis-logs) | N | Y | +| [Registry Logs](#registry-logs) | N | Y | +| [Workhorse Logs](#workhorse-logs) | Y | Y | + ## `production_json.log` This file lives in `/var/log/gitlab/gitlab-rails/production_json.log` for @@ -275,7 +311,7 @@ installations from source. It contains the JSON version of the logs in `application.log` like the example below: -``` json +```json { "severity":"INFO", "time":"2020-01-14T13:35:15.466Z", @@ -302,7 +338,7 @@ It contains information about [integrations](../user/project/integrations/overvi { "severity":"ERROR", "time":"2018-09-06T14:56:20.439Z", - "service_class":"JiraService", + "service_class":"Integrations::Jira", "project_id":8, "project_path":"h5bp/html5-boilerplate", "message":"Error sending message", @@ -312,7 +348,7 @@ It contains information about [integrations](../user/project/integrations/overvi { "severity":"INFO", "time":"2018-09-06T17:15:16.365Z", - "service_class":"JiraService", + "service_class":"Integrations::Jira", "project_id":3, "project_path":"namespace2/project2", "message":"Successfully posted", @@ -611,41 +647,6 @@ This file lives in `/var/log/gitlab/puma/puma_stderr.log` for Omnibus GitLab packages, or in `/home/git/gitlab/log/puma_stderr.log` for installations from source. -## Unicorn Logs - -Starting with GitLab 13.0, Puma is the default web server used in GitLab -all-in-one package based installations, and GitLab Helm chart deployments. - -### `unicorn_stdout.log` - -This file lives in `/var/log/gitlab/unicorn/unicorn_stdout.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stdout.log` for -for installations from source. - -### `unicorn_stderr.log` - -This file lives in `/var/log/gitlab/unicorn/unicorn_stderr.log` for -Omnibus GitLab packages or in `/home/git/gitlab/log/unicorn_stderr.log` for -for installations from source. - -These logs contain all information about the state of Unicorn processes at any given time. - -```plaintext -I, [2015-02-13T06:14:46.680381 #9047] INFO -- : Refreshing Gem list -I, [2015-02-13T06:14:56.931002 #9047] INFO -- : listening on addr=127.0.0.1:8080 fd=12 -I, [2015-02-13T06:14:56.931381 #9047] INFO -- : listening on addr=/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket fd=13 -I, [2015-02-13T06:14:56.936638 #9047] INFO -- : master process ready -I, [2015-02-13T06:14:56.946504 #9092] INFO -- : worker=0 spawned pid=9092 -I, [2015-02-13T06:14:56.946943 #9092] INFO -- : worker=0 ready -I, [2015-02-13T06:14:56.947892 #9094] INFO -- : worker=1 spawned pid=9094 -I, [2015-02-13T06:14:56.948181 #9094] INFO -- : worker=1 ready -W, [2015-02-13T07:16:01.312916 #9094] WARN -- : #: worker (pid: 9094) exceeds memory limit (320626688 bytes > 247066940 bytes) -W, [2015-02-13T07:16:01.313000 #9094] WARN -- : Unicorn::WorkerKiller send SIGQUIT (pid: 9094) alive: 3621 sec (trial 1) -I, [2015-02-13T07:16:01.530733 #9047] INFO -- : reaped # worker=1 -I, [2015-02-13T07:16:01.534501 #13379] INFO -- : worker=1 spawned pid=13379 -I, [2015-02-13T07:16:01.534848 #13379] INFO -- : worker=1 ready -``` - ## `repocheck.log` This file lives in `/var/log/gitlab/gitlab-rails/repocheck.log` for @@ -774,7 +775,7 @@ When enabled, access logs are generated in packages or in `/home/git/gitlab/log/sidekiq_exporter.log` for installations from source. -If Prometheus metrics and the Web Exporter are both enabled, Puma/Unicorn +If Prometheus metrics and the Web Exporter are both enabled, Puma starts a Web server and listen to the defined port (default: `8083`), and access logs are generated: @@ -824,7 +825,7 @@ Line breaks have been added to the following example line for clarity: This file logs the information about exceptions being tracked by `Gitlab::ErrorTracking`, which provides a standard and consistent way of -[processing rescued exceptions](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/logging.md#exception-handling). This file is stored in: +[processing rescued exceptions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/logging.md#exception-handling). This file is stored in: - `/var/log/gitlab/gitlab-rails/exceptions_json.log` for Omnibus GitLab packages. - `/home/git/gitlab/log/exceptions_json.log` for installations from source. diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index 7fd383eabae..c73a49287db 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -35,8 +35,8 @@ There are three ways to enable Maintenance Mode as an administrator: - [**Rails console**](../operations/rails_console.md#starting-a-rails-console-session): ```ruby - ::Gitlab::CurrentSettings.update_attributes!(maintenance_mode: true) - ::Gitlab::CurrentSettings.update_attributes!(maintenance_mode_message: "New message") + ::Gitlab::CurrentSettings.update!(maintenance_mode: true) + ::Gitlab::CurrentSettings.update!(maintenance_mode_message: "New message") ``` ## Disable Maintenance Mode @@ -143,25 +143,25 @@ is turned off. ### Deployments -Deployments won't go through because pipelines will be unfinished. +Deployments don't go through because pipelines are unfinished. It is recommended to disable auto deploys during Maintenance Mode, and enable them once it is disabled. #### Terraform integration -Terraform integration depends on running CI pipelines, hence it will be blocked. +Terraform integration depends on running CI pipelines, hence it is blocked. ### Container Registry -`docker push` will fail with this error: `denied: requested access to the resource is denied`, but `docker pull` will work. +`docker push` fails with this error: `denied: requested access to the resource is denied`, but `docker pull` works. ### Package Registry -Package Registry will allow you to install but not publish packages. +Package Registry allows you to install but not publish packages. ### Background jobs -Background jobs (cron jobs, Sidekiq) will continue running as is, because background jobs are not automatically disabled. +Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled. [During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node), it is recommended that you disable all cron jobs except for those related to Geo. @@ -170,34 +170,34 @@ You can monitor queues and disable jobs in **Admin Area > Monitoring > Backgroun ### Incident management -[Incident management](../../operations/incident_management/index.md) functions will be limited. The creation of [alerts](../../operations/incident_management/alerts.md) and [incidents](../../operations/incident_management/incidents.md#incident-creation) will be paused entirely. Notifications and paging on alerts and incidents will therefore be disabled. +[Incident management](../../operations/incident_management/index.md) functions are limited. The creation of [alerts](../../operations/incident_management/alerts.md) and [incidents](../../operations/incident_management/incidents.md#incident-creation) are paused entirely. Notifications and paging on alerts and incidents are therefore disabled. ### Feature flags - [Development feature flags](../../development/feature_flags/index.md) cannot be turned on or off through the API, but can be toggled through the Rails console. -- [The feature flag service](../../operations/feature_flags.md) will respond to feature flag checks but feature flags cannot be toggled +- [The feature flag service](../../operations/feature_flags.md) responds to feature flag checks but feature flags cannot be toggled ### Geo secondaries -When primary is in Maintenance Mode, secondary will also automatically go into Maintenance Mode. +When primary is in Maintenance Mode, secondary also automatically goes into Maintenance Mode. It is important that you do not disable replication before enabling Maintenance Mode. -Replication and verification will continue to work but proxied Git pushes to primary will not work. +Replication and verification continues to work but proxied Git pushes to primary do not work. ### Secure features -Features that depend on creating issues or creating or approving Merge Requests, will not work. +Features that depend on creating issues or creating or approving Merge Requests, do not work. -Exporting a vulnerability list from a Vulnerability Report page will not work. +Exporting a vulnerability list from a Vulnerability Report page does not work. -Changing the status on a finding or vulnerability object will not work, even though no error is shown in the UI. +Changing the status on a finding or vulnerability object does not work, even though no error is shown in the UI. SAST and Secret Detection cannot be initiated because they depend on passing CI jobs to create artifacts. ## An example use case: a planned failover -In the use case of [a planned failover](../geo/disaster_recovery/planned_failover.md), a few writes in the primary database are acceptable, since they will be replicated quickly and are not significant in number. +In the use case of [a planned failover](../geo/disaster_recovery/planned_failover.md), a few writes in the primary database are acceptable, since they are replicated quickly and are not significant in number. For the same reason we don't automatically block background jobs when Maintenance Mode is enabled. diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 8d3c8555660..b1ec74c2f40 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -20,7 +20,8 @@ GitLab instance. All administrators at the time of creation of the project and group are added as maintainers of the group and project, and as an administrator, you can -add new members to the group to give them maintainer access to the project. +add new members to the group to give them the [Maintainer role](../../../user/permissions.md) for +the project. This project is used to self monitor your GitLab instance. The metrics dashboard of the project shows some basic resource usage charts, such as CPU and memory usage @@ -32,9 +33,12 @@ metrics exposed by the [GitLab exporter](../prometheus/gitlab_metrics.md#metrics ## Creating the self monitoring project -1. Go to **Admin Area > Settings > Metrics and profiling** and expand the **Self monitoring** section. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling** and expand **Self monitoring**. 1. Toggle the **Create Project** button on. -1. Once your GitLab instance creates the project, GitLab displays a link to the project in the text above the **Create Project** toggle. You can also find it under **Projects > Your projects**. +1. After your GitLab instance creates the project, GitLab displays a link to the + project in the text above the **Create Project** toggle. You can also find it + from the top bar by selecting **Menu > Project**, then selecting **Your projects**. ## Deleting the self monitoring project @@ -42,7 +46,8 @@ WARNING: Deleting the self monitoring project removes any changes made to the project. If you create the project again, it's created in its default state. -1. Go to **Admin Area > Settings > Metrics and profiling** and expand the **Self monitoring** section. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, go to **Settings > Metrics and profiling** and expand **Self monitoring**. 1. Toggle the **Create Project** button off. 1. In the confirmation dialog that opens, click **Delete project**. It can take a few seconds for it to be deleted. diff --git a/doc/administration/monitoring/performance/gitlab_configuration.md b/doc/administration/monitoring/performance/gitlab_configuration.md index 6e7557854ad..e8abe2708c7 100644 --- a/doc/administration/monitoring/performance/gitlab_configuration.md +++ b/doc/administration/monitoring/performance/gitlab_configuration.md @@ -9,7 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab Performance Monitoring is disabled by default. To enable it and change any of its settings: -1. Go to **Admin Area > Settings > Metrics and profiling** +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling** (`/admin/application_settings/metrics_and_profiling`). 1. Add the necessary configuration changes. 1. Restart all GitLab for the changes to take effect: diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index ac322f3e1ef..3b82b0e4bb8 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -62,8 +62,9 @@ repository. After setting up Grafana, you can enable a link to access it easily from the GitLab sidebar: -1. Go to the **Admin Area > Settings > Metrics and profiling**. -1. Expand **Metrics - Grafana**. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling** + and expand **Metrics - Grafana**. 1. Check the **Enable access to Grafana** checkbox. 1. Configure the **Grafana URL**: - *If Grafana is enabled through Omnibus GitLab and on the same server,* @@ -71,7 +72,7 @@ GitLab sidebar: - *Otherwise,* enter the full URL of the Grafana instance. 1. Click **Save changes**. -GitLab displays your link in the **Admin Area > Monitoring > Metrics Dashboard**. +GitLab displays your link in the **Menu > Admin > Monitoring > Metrics Dashboard**. ## Security Update diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 15c54a36f6c..f3db6ac9f03 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -69,6 +69,5 @@ The following environment variables are recognized: - `DATABASE_SAMPLER_INTERVAL_SECONDS` - `ACTION_CABLE_SAMPLER_INTERVAL_SECONDS` - `PUMA_SAMPLER_INTERVAL_SECONDS` -- `UNICORN_SAMPLER_INTERVAL_SECONDS` - `THREADS_SAMPLER_INTERVAL_SECONDS` - `GLOBAL_SEARCH_SAMPLER_INTERVAL_SECONDS` diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index dd43c7d6fbb..1125547f13f 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Performance Bar **(FREE SELF)** -> The **Stats** field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271551) in GitLab SaaS 13.9. +> The **Stats** field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271551) in GitLab 13.9. +> The **Memory** field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330736) in GitLab 14.0. You can display the GitLab Performance Bar to see statistics for the performance of a page. When activated, it looks as follows: @@ -40,9 +41,11 @@ From left to right, it displays: Time until something was visible to the user. - [**DomContentLoaded**](https://developers.google.com/web/fundamentals/performance/critical-rendering-path/measure-crp) Event. - **Total number of requests** the page loaded. -- **Trace**: If Jaeger is integrated, **Trace** links to a Jaeger tracing page +- **Memory**: the amount of memory consumed and objects allocated during the selected request. + Select it to display a window with more details. +- **Trace**: if Jaeger is integrated, **Trace** links to a Jaeger tracing page with the current request's `correlation_id` included. -- **+**: A link to add a request's details to the performance bar. The request +- **+**: a link to add a request's details to the performance bar. The request can be added by its full URL (authenticated as the current user), or by the value of its `X-Request-Id` header. - **Download**: a link to download the raw JSON used to generate the Performance Bar reports. @@ -52,6 +55,11 @@ From left to right, it displays: - **Stats** (optional): if the `GITLAB_PERFORMANCE_BAR_STATS_URL` environment variable is set, this URL is displayed in the bar. In GitLab 13.9 and later, used only in GitLab SaaS. +NOTE: +Not all indicators are available in all environments. For instance, the memory view +requires to run Ruby with [specific patches](https://gitlab.com/gitlab-org/gitlab-build-images/-/blob/master/patches/ruby/2.7.2/thread-memory-allocations-2.7.patch) applied. +When running GitLab locally using the GDK this is typically not the case and the memory view cannot be used. + ## Request warnings Requests that exceed predefined limits display a warning **{warning}** icon and @@ -68,12 +76,13 @@ appears next to requests with warnings. ## Enable the Performance Bar via the Admin Area -The GitLab Performance Bar is disabled by default. To enable it for a given group: +The GitLab Performance Bar is disabled by default for non-administrators. To enable it +for a given group: 1. Sign in as a user with Administrator [permissions](../../../user/permissions.md). -1. In the menu bar, click **Admin Area**. -1. Go to **Settings > Metrics and profiling** - (`admin/application_settings/metrics_and_profiling`), and expand the section +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling** + (`admin/application_settings/metrics_and_profiling`), and expand **Profiling - Performance bar**. 1. Click **Enable access to the Performance Bar**. 1. In the **Allowed group** field, provide the full path of the group allowed diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index 15a58456e05..ebdca8d3960 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w To profile a request: -1. Sign in to GitLab as a user with Administrator or Maintainer [permissions](../../../user/permissions.md). +1. Sign in to GitLab as an Administrator or a user with the [Maintainer role](../../../user/permissions.md). 1. In the navigation bar, click **Admin area**. 1. Go to **Monitoring > Requests Profiles**. 1. In the **Requests Profiles** section, copy the token. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index f29db9ead38..7e72f6ed7df 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -9,7 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w To enable the GitLab Prometheus metrics: 1. Log into GitLab as a user with [administrator permissions](../../../user/permissions.md). -1. Go to **Admin Area > Settings > Metrics and profiling**. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling**. 1. Find the **Metrics - Prometheus** section, and click **Enable Prometheus Metrics**. 1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect. @@ -58,7 +59,7 @@ The following metrics are available: | `gitlab_transaction_cache_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (aggregate) | | | `gitlab_transaction_cache_read_hit_count_total` | Counter | 10.2 | Counter for cache hits for Rails cache calls | `controller`, `action` | | `gitlab_transaction_cache_read_miss_count_total` | Counter | 10.2 | Counter for cache misses for Rails cache calls | `controller`, `action` | -| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (`gitlab_transaction_*` metrics) | `controller`, `action` | +| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for successful requests (`gitlab_transaction_*` metrics) | `controller`, `action` | | `gitlab_transaction_event_build_found_total` | Counter | 9.4 | Counter for build found for API /jobs/request | | | `gitlab_transaction_event_build_invalid_total` | Counter | 9.4 | Counter for build invalid due to concurrency conflict for API /jobs/request | | | `gitlab_transaction_event_build_not_found_cached_total` | Counter | 9.4 | Counter for cached response of build not found for API /jobs/request | | @@ -91,7 +92,7 @@ The following metrics are available: | `gitlab_transaction_view_duration_total` | Counter | 9.4 | Duration for views | `controller`, `action`, `view` | | `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | `controller`, `action`, `view` | | `http_requests_total` | Counter | 9.4 | Rack request count | `method`, `status` | -| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | `method` | +| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware for successful requests | `method` | | `gitlab_transaction_db_count_total` | Counter | 13.1 | Counter for total number of SQL calls | `controller`, `action` | | `gitlab_transaction_db__count_total` | Counter | 13.10 | Counter for total number of SQL calls, grouped by database roles (primary/replica) | `controller`, `action` | | `gitlab_transaction_db_write_count_total` | Counter | 13.1 | Counter for total number of write SQL calls | `controller`, `action` | @@ -130,6 +131,8 @@ The following metrics are available: | `gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds` | Histogram | 13.12 | Time in seconds it takes to process Security Policies in CI/CD pipeline | | | `gitlab_ci_difference_live_vs_actual_minutes` | Histogram | 13.12 | Difference between CI minute consumption counted while jobs were running (live) vs when jobs are complete (actual). Used to enforce CI minute consumption limits on long running jobs. | `plan` | | `gitlab_spamcheck_request_duration_seconds` | Histogram | 13.12 | The duration for requests between Rails and the anti-spam engine | | +| `service_desk_thank_you_email` | Counter | 14.0 | Total number of email responses to new service desk emails | | +| `service_desk_new_note_email` | Counter | 14.0 | Total number of email notifications on new service desk comment | | ## Metrics controlled by a feature flag @@ -227,11 +230,15 @@ configuration option in `gitlab.yml`. These metrics are served from the | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | | `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `geo_merge_request_diffs` | Gauge | 13.4 | Number of merge request diffs on primary | `url` | -| `geo_merge_request_diffs_checksummed` | Gauge | 13.4 | Number of merge request diffs checksummed on primary | `url` | +| `geo_merge_request_diffs_checksum_total` | Gauge | 13.12 | Number of merge request diffs tried to checksum on primary | `url` | +| `geo_merge_request_diffs_checksummed` | Gauge | 13.4 | Number of merge request diffs successfully checksummed on primary | `url` | | `geo_merge_request_diffs_checksum_failed` | Gauge | 13.4 | Number of merge request diffs failed to calculate the checksum on primary | `url` | | `geo_merge_request_diffs_synced` | Gauge | 13.4 | Number of syncable merge request diffs synced on secondary | `url` | | `geo_merge_request_diffs_failed` | Gauge | 13.4 | Number of syncable merge request diffs failed to sync on secondary | `url` | | `geo_merge_request_diffs_registry` | Gauge | 13.4 | Number of merge request diffs in the registry | `url` | +| `geo_merge_request_diffs_verification_total` | Gauge | 13.12 | Number of merge request diffs verifications tried on secondary | `url` | +| `geo_merge_request_diffs_verified` | Gauge | 13.12 | Number of merge request diffs verified on secondary | `url` | +| `geo_merge_request_diffs_verification_failed` | Gauge | 13.12 | Number of merge request diffs verifications failed on secondary | `url` | | `geo_snippet_repositories` | Gauge | 13.4 | Number of snippets on primary | `url` | | `geo_snippet_repositories_checksummed` | Gauge | 13.4 | Number of snippets checksummed on primary | `url` | | `geo_snippet_repositories_checksum_failed` | Gauge | 13.4 | Number of snippets failed to calculate the checksum on primary | `url` | @@ -308,20 +315,8 @@ Some basic Ruby runtime metrics are available: | `ruby_process_proportional_memory_bytes` | Gauge | 13.0 | Memory usage by process (PSS/Proportional Set Size) | | `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time | -## Unicorn Metrics - -Unicorn specific metrics, when Unicorn is used. - -| Metric | Type | Since | Description | -|:-----------------------------|:------|:------|:---------------------------------------------------| -| `unicorn_active_connections` | Gauge | 11.0 | The number of active Unicorn connections (workers) | -| `unicorn_queued_connections` | Gauge | 11.0 | The number of queued Unicorn connections | -| `unicorn_workers` | Gauge | 12.0 | The number of Unicorn workers | - ## Puma Metrics -When Puma is used instead of Unicorn, the following metrics are available: - | Metric | Type | Since | Description | |:--------------------------------- |:------- |:----- |:----------- | | `puma_workers` | Gauge | 12.0 | Total number of workers | @@ -352,8 +347,8 @@ instance (`cache`, `shared_state` etc.). ## Metrics shared directory The GitLab Prometheus client requires a directory to store metrics data shared between multi-process services. -Those files are shared among all instances running under Unicorn server. -The directory must be accessible to all running Unicorn's processes, or +Those files are shared among all instances running under Puma server. +The directory must be accessible to all running Puma's processes, or metrics can't function correctly. This directory's location is configured using environment variable `prometheus_multiproc_dir`. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 035c5b3ee7e..dd402f800e3 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -122,44 +122,28 @@ The steps below are the minimum necessary to configure a Monitoring node running 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby + roles ['monitoring_role'] + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana + # Grafana grafana['enable'] = true grafana['admin_password'] = 'toomanysecrets' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus consul['enable'] = true consul['monitoring_service_discovery'] = true - - # The addresses can be IPs or FQDNs - consul['configuration'] = { - retry_join: %w(10.0.0.1 10.0.0.2 10.0.0.3), + consul['configuration'] = { + retry_join: %w(10.0.0.1 10.0.0.2 10.0.0.3), # The addresses can be IPs or FQDNs } - # Disable all other services - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false + # Nginx - For Grafana access nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false ``` 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. @@ -227,7 +211,7 @@ To use an external Prometheus server: gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] ``` -1. On **all** GitLab Rails(Puma/Unicorn, Sidekiq) servers, set the Prometheus server IP address and listen port. For example: +1. On **all** GitLab Rails(Puma, Sidekiq) servers, set the Prometheus server IP address and listen port. For example: ```ruby gitlab_rails['prometheus_address'] = '192.168.0.1:9090' diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index c49a2c20ed2..e53f2af3440 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -5,11 +5,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Using NFS with GitLab +# Using NFS with GitLab **(FREE SELF)** NFS can be used as an alternative for object storage but this isn't typically -recommended for performance reasons. Note however it is required for [GitLab -Pages](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/196). +recommended for performance reasons. For data objects such as LFS, Uploads, Artifacts, etc., an [Object Storage service](object_storage.md) is recommended over NFS where possible, due to better performance. @@ -17,7 +16,7 @@ is recommended over NFS where possible, due to better performance. File system performance can impact overall GitLab performance, especially for actions that read or write to Git repositories. For steps you can use to test file system performance, see -[File system Performance Benchmarking](operations/filesystem_benchmarking.md). +[File System Performance Benchmarking](operations/filesystem_benchmarking.md). ## Gitaly and NFS deprecation @@ -445,11 +444,11 @@ In case of NFS-related problems, it can be helpful to trace the file system requests that are being made by using `perf`: ```shell -sudo perf trace -e 'nfs4:*' -p $(pgrep -fd ',' puma && pgrep -fd ',' unicorn) +sudo perf trace -e 'nfs4:*' -p $(pgrep -fd ',' puma) ``` On Ubuntu 16.04, use: ```shell -sudo perf trace --no-syscalls --event 'nfs4:*' -p $(pgrep -fd ',' puma && pgrep -fd ',' unicorn) +sudo perf trace --no-syscalls --event 'nfs4:*' -p $(pgrep -fd ',' puma) ``` diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index d133e8c3721..f1025bd1846 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Object Storage +# Object storage **(FREE SELF)** GitLab supports using an object storage service for holding numerous types of data. It's recommended over NFS and @@ -29,7 +29,7 @@ GitLab has been tested on a number of object storage providers: - Dell EMC ECS: Prior to GitLab 13.3, there is a [known bug in GitLab Workhorse that prevents HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). - Be sure to upgrade to GitLab v13.3.0 or above if you use S3 storage with this hardware. + Be sure to upgrade to GitLab 13.3.0 or above if you use S3 storage with this hardware. - Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying). @@ -47,7 +47,7 @@ For more information on the differences and to transition from one form to anoth ### Consolidated object storage configuration -> Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4368). +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4368) in GitLab 13.2. Using the consolidated object storage configuration has a number of advantages: @@ -209,13 +209,13 @@ gitlab_rails['object_store']['connection'] = { } ``` -| Setting | Description | -|---------|-------------| -| `enabled` | Enable/disable object storage | -| `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | -| `connection` | Various [connection options](#connection-settings) described below | -| `storage_options` | Options to use when saving new objects, such as [server side encryption](#server-side-encryption-headers). Introduced in GitLab 13.3 | -| `objects` | [Object-specific configuration](#object-specific-configuration) +| Setting | Description | +|-------------------|-----------------------------------| +| `enabled` | Enable or disable object storage. | +| `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | +| `connection` | Various [connection options](#connection-settings) described below. | +| `storage_options` | Options to use when saving new objects, such as [server side encryption](#server-side-encryption-headers). Introduced in GitLab 13.3. | +| `objects` | [Object-specific configuration](#object-specific-configuration). | ### Connection settings @@ -226,27 +226,27 @@ in the `connection` setting. The connection settings match those provided by [fog-aws](https://github.com/fog/fog-aws): -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `AWS` for compatible hosts | `AWS` | -| `aws_access_key_id` | AWS credentials, or compatible | | -| `aws_secret_access_key` | AWS credentials, or compatible | | -| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | -| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | -| `region` | AWS region. | | -| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | -| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false` | -| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys | `false` +| Setting | Description | Default | +|---------------------------------|------------------------------------|---------| +| `provider` | Always `AWS` for compatible hosts. | `AWS` | +| `aws_access_key_id` | AWS credentials, or compatible. | | +| `aws_secret_access_key` | AWS credentials, or compatible. | | +| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | +| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | +| `region` | AWS region. | | +| `host` | S3 compatible host for when not using AWS. For example, `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | +| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | +| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false`. | +| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys. | `false` | #### Oracle Cloud S3 connection settings Note that Oracle Cloud S3 must be sure to use the following settings: -| Setting | Value | -|---------|-------| +| Setting | Value | +|---------------------------------|---------| | `enable_signature_v4_streaming` | `false` | -| `path_style` | `true` | +| `path_style` | `true` | If `enable_signature_v4_streaming` is set to `true`, you may see the following error in `production.log`: @@ -259,13 +259,13 @@ STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported Here are the valid connection parameters for GCS: -| Setting | Description | example | -|---------|-------------|---------| -| `provider` | The provider name | `Google` | -| `google_project` | GCP project name | `gcp-project-12345` | -| `google_client_email` | The email address of the service account | `foo@gcp-project-12345.iam.gserviceaccount.com` | -| `google_json_key_location` | The JSON key path | `/path/to/gcp-project-12345-abcde.json` | -| `google_application_default` | Set to `true` to use [Google Cloud Application Default Credentials](https://cloud.google.com/docs/authentication/production#automatically) to locate service account credentials. | +| Setting | Description | Example | +|------------------------------|-------------------|---------| +| `provider` | Provider name. | `Google` | +| `google_project` | GCP project name. | `gcp-project-12345` | +| `google_client_email` | Email address of the service account. | `foo@gcp-project-12345.iam.gserviceaccount.com` | +| `google_json_key_location` | JSON key path. | `/path/to/gcp-project-12345-abcde.json` | +| `google_application_default` | Set to `true` to use [Google Cloud Application Default Credentials](https://cloud.google.com/docs/authentication/production#automatically) to locate service account credentials. | | The service account must have permission to access the bucket. Learn more in Google's @@ -328,12 +328,12 @@ The following are the valid connection parameters for Azure. Read the [Azure Blob storage documentation](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) to learn more. -| Setting | Description | Example | -|---------|-------------|---------| -| `provider` | Provider name | `AzureRM` | -| `azure_storage_account_name` | Name of the Azure Blob Storage account used to access the storage | `azuretest` | -| `azure_storage_access_key` | Storage account access key used to access the container. This is typically a secret, 512-bit encryption key encoded in base64. | `czV2OHkvQj9FKEgrTWJRZVRoV21ZcTN0Nnc5eiRDJkYpSkBOY1JmVWpYbjJy\nNHU3eCFBJUQqRy1LYVBkU2dWaw==\n` | -| `azure_storage_domain` | Domain name used to contact the Azure Blob Storage API (optional). Defaults to `blob.core.windows.net`. Set this if you are using Azure China, Azure Germany, Azure US Government, or some other custom Azure domain. | `blob.core.windows.net` | +| Setting | Description | Example | +|------------------------------|----------------|-----------| +| `provider` | Provider name. | `AzureRM` | +| `azure_storage_account_name` | Name of the Azure Blob Storage account used to access the storage. | `azuretest` | +| `azure_storage_access_key` | Storage account access key used to access the container. This is typically a secret, 512-bit encryption key encoded in base64. | `czV2OHkvQj9FKEgrTWJRZVRoV21ZcTN0Nnc5eiRDJkYpSkBOY1JmVWpYbjJy\nNHU3eCFBJUQqRy1LYVBkU2dWaw==\n` | +| `azure_storage_domain` | Domain name used to contact the Azure Blob Storage API (optional). Defaults to `blob.core.windows.net`. Set this if you are using Azure China, Azure Germany, Azure US Government, or some other custom Azure domain. | `blob.core.windows.net` | ##### Azure example (consolidated form) @@ -382,15 +382,15 @@ consolidated form, see the [S3 settings](#s3-compatible-connection-settings). Here are the valid connection settings for the Swift API, provided by [fog-openstack](https://github.com/fog/fog-openstack): -| Setting | Description | Default | -|---------|-------------|---------| -| `provider` | Always `OpenStack` for compatible hosts | `OpenStack` | -| `openstack_username` | OpenStack username | | -| `openstack_api_key` | OpenStack API key | | +| Setting | Description | Default | +|--------------------------|----------------------|---------| +| `provider` | Always `OpenStack` for compatible hosts. | `OpenStack` | +| `openstack_username` | OpenStack username. | | +| `openstack_api_key` | OpenStack API key. | | | `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | -| `openstack_auth_url` | OpenStack authentication endpoint | | -| `openstack_region` | OpenStack region | | -| `openstack_tenant` | OpenStack tenant ID | +| `openstack_auth_url` | OpenStack authentication endpoint | | +| `openstack_region` | OpenStack region. | | +| `openstack_tenant` | OpenStack tenant ID. | | #### Rackspace Cloud Files @@ -400,13 +400,13 @@ Rackspace Cloud, provided by [fog-rackspace](https://github.com/fog/fog-rackspac This isn't compatible with the consolidated object storage form. Rackspace Cloud is supported only with the storage-specific form. -| Setting | Description | example | -|---------|-------------|---------| -| `provider` | The provider name | `Rackspace` | -| `rackspace_username` | The username of the Rackspace account with access to the container | `joe.smith` | -| `rackspace_api_key` | The API key of the Rackspace account with access to the container | `ABC123DEF456ABC123DEF456ABC123DE` | -| `rackspace_region` | The Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://docs.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/) | `iad` | -| `rackspace_temp_url_key` | The private key you have set in the Rackspace API for [temporary URLs](https://docs.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl). | `ABC123DEF456ABC123DEF456ABC123DE` | +| Setting | Description | Example | +|--------------------------|----------------|-------------| +| `provider` | Provider name. | `Rackspace` | +| `rackspace_username` | Username of the Rackspace account with access to the container. | `joe.smith` | +| `rackspace_api_key` | API key of the Rackspace account with access to the container. | `ABC123DEF456ABC123DEF456ABC123DE` | +| `rackspace_region` | Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://docs.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/). | `iad` | +| `rackspace_temp_url_key` | Private key you set in the Rackspace API for [temporary URLs](https://docs.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl). | `ABC123DEF456ABC123DEF456ABC123DE` | Regardless of whether the container has public access enabled or disabled, Fog uses the TempURL method to grant access to LFS objects. If you see error @@ -465,24 +465,24 @@ gitlab_rails['object_store']['objects']['pages']['bucket'] = 'pages' This is the list of valid `objects` that can be used: -| Type | Description | -|--------------------|---------------| -| `artifacts` | [CI artifacts](job_artifacts.md) | -| `external_diffs` | [Merge request diffs](merge_request_diffs.md) | -| `uploads` | [User uploads](uploads.md) | -| `lfs` | [Git Large File Storage objects](lfs/index.md) | -| `packages` | [Project packages (e.g. PyPI, Maven, NuGet, etc.)](packages/index.md) | -| `dependency_proxy` | [GitLab Dependency Proxy](packages/dependency_proxy.md) | -| `terraform_state` | [Terraform state files](terraform_state.md) | -| `pages` | [GitLab Pages](pages/index.md) | +| Type | Description | +|--------------------|----------------------------------------------------------------------------| +| `artifacts` | [CI artifacts](job_artifacts.md) | +| `external_diffs` | [Merge request diffs](merge_request_diffs.md) | +| `uploads` | [User uploads](uploads.md) | +| `lfs` | [Git Large File Storage objects](lfs/index.md) | +| `packages` | [Project packages (for example, PyPI, Maven, or NuGet)](packages/index.md) | +| `dependency_proxy` | [GitLab Dependency Proxy](packages/dependency_proxy.md) | +| `terraform_state` | [Terraform state files](terraform_state.md) | +| `pages` | [GitLab Pages](pages/index.md) | Within each object type, three parameters can be defined: -| Setting | Required? | Description | -|------------------|-----------|-------------| -| `bucket` | Yes | The bucket name for the object storage. | -| `enabled` | No | Overrides the common parameter | -| `proxy_download` | No | Overrides the common parameter | +| Setting | Required? | Description | +|------------------|------------------------|-------------------------------------| +| `bucket` | **{check-circle}** Yes | Bucket name for the object storage. | +| `enabled` | **{dotted-circle}** No | Overrides the common parameter. | +| `proxy_download` | **{dotted-circle}** No | Overrides the common parameter. | #### Selectively disabling object storage @@ -542,21 +542,21 @@ original configuration (for example, `artifacts_object_store_enabled`, or For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides: -|Object storage type|Supported by consolidated configuration?| -|-------------------|----------------------------------------| -| [Backups](../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage) | No | -| [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | Yes | -| [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | -| [Uploads](uploads.md#using-object-storage) | Yes | -| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | No | -| [Merge request diffs](merge_request_diffs.md#using-object-storage) | Yes | -| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | -| [Packages](packages/index.md#using-object-storage) (optional feature) | Yes | -| [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM SELF)** | Yes | -| [Pseudonymizer](pseudonymizer.md#configuration) (optional feature) **(ULTIMATE SELF)** | No | -| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | -| [Terraform state files](terraform_state.md#using-object-storage) | Yes | -| [GitLab Pages content](pages/index.md#using-object-storage) | Yes | +| Object storage type | Supported by consolidated configuration? | +|---------------------|------------------------------------------| +| [Backups](../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage) | **{dotted-circle}** No | +| [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | **{check-circle}** Yes | +| [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | **{check-circle}** Yes | +| [Uploads](uploads.md#using-object-storage) | **{check-circle}** Yes | +| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | +| [Merge request diffs](merge_request_diffs.md#using-object-storage) | **{check-circle}** Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| **{dotted-circle}** No | +| [Packages](packages/index.md#using-object-storage) (optional feature) | **{check-circle}** Yes | +| [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM SELF)** | **{check-circle}** Yes | +| [Pseudonymizer](pseudonymizer.md#configuration) (optional feature) **(ULTIMATE SELF)** | **{dotted-circle}** No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | +| [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | +| [GitLab Pages content](pages/index.md#using-object-storage) | **{check-circle}** Yes | ### Other alternatives to file system storage @@ -618,20 +618,20 @@ This can result in some of the following problems: - If GitLab is using non-secure HTTP to access the object storage, clients may generate `https->http` downgrade errors and refuse to process the redirect. The solution to this -is for GitLab to use HTTPS. LFS, for example, will generate this error: +is for GitLab to use HTTPS. LFS, for example, generates this error: ```plaintext LFS: lfsapi/client: refusing insecure redirect, https->http ``` -- Clients will need to trust the certificate authority that issued the object storage +- Clients need to trust the certificate authority that issued the object storage certificate, or may return common TLS errors such as: ```plaintext x509: certificate signed by unknown authority ``` -- Clients will need network access to the object storage. +- Clients need network access to the object storage. Network firewalls could block access. Errors that might result if this access is not in place include: @@ -667,7 +667,7 @@ The first option is recommended for MinIO. Otherwise, the is to use the `--compat` parameter on the server. Without consolidated object store configuration or instance profiles enabled, -GitLab Workhorse will upload files to S3 using pre-signed URLs that do +GitLab Workhorse uploads files to S3 using pre-signed URLs that do not have a `Content-MD5` HTTP header computed for them. To ensure data is not corrupted, Workhorse checks that the MD5 hash of the data sent equals the ETag header returned from the S3 server. When encryption is @@ -683,7 +683,7 @@ eliminates the need to compare ETag headers returned from the S3 server. Instead of supplying AWS access and secret keys in object storage configuration, GitLab can be configured to use IAM roles to set up an [Amazon instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). -When this is used, GitLab will fetch temporary credentials each time an +When this is used, GitLab fetches temporary credentials each time an S3 bucket is accessed, so no hard-coded values are needed in the configuration. @@ -709,10 +709,10 @@ only encrypted objects are uploaded](https://aws.amazon.com/premiumsupport/knowl To do this, you must configure GitLab to send the proper encryption headers in the `storage_options` configuration section: -| Setting | Description | -|-------------------------------------|-------------| -| `server_side_encryption` | Encryption mode (`AES256` or `aws:kms`) | -| `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) | +| Setting | Description | +|-------------------------------------|------------------------------------------| +| `server_side_encryption` | Encryption mode (`AES256` or `aws:kms`). | +| `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). | As with the case for default encryption, these options only work when the Workhorse S3 client is enabled. One of the following two conditions @@ -721,7 +721,7 @@ must be fulfilled: - `use_iam_profile` is `true` in the connection settings. - Consolidated object storage settings are in use. -[ETag mismatch errors](#etag-mismatch) will occur if server side +[ETag mismatch errors](#etag-mismatch) occur if server side encryption headers are used without enabling the Workhorse S3 client. ##### Disabling the feature diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 6513a4ed4c8..194dd8f39e2 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -25,7 +25,7 @@ NOTE: The instructions below must be modified in accordance with your configuration settings if you have used the advanced Redis settings outlined in -[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/config/README.md). +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/README.md). First we define a shell function with the proper Redis connection details. diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 6b8d6f3bf1e..ed89d11da75 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -18,8 +18,8 @@ The information in this page applies only to Omnibus GitLab. For a list of the existing Sidekiq queues, check the following files: -- [Queues for both GitLab Community and Enterprise Editions](https://gitlab.com/gitlab-org/gitlab/blob/master/app/workers/all_queues.yml) -- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/workers/all_queues.yml) +- [Queues for both GitLab Community and Enterprise Editions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) +- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml) Each entry in the above files represents a queue on which Sidekiq processes can be started. @@ -38,11 +38,11 @@ To start multiple processes: process, and values in each item determine the queues it works on. For example, the following setting creates three Sidekiq processes, one to run on - `elastic_indexer`, one to run on `mailers`, and one process running on all queues: + `elastic_commit_indexer`, one to run on `mailers`, and one process running on all queues: ```ruby sidekiq['queue_groups'] = [ - "elastic_indexer", + "elastic_commit_indexer", "mailers", "*" ] @@ -53,7 +53,7 @@ To start multiple processes: ```ruby sidekiq['queue_groups'] = [ - "elastic_indexer, elastic_commit_indexer", + "elastic_commit_indexer, elastic_association_indexer", "mailers", "*" ] @@ -70,11 +70,11 @@ To start multiple processes: ] ``` - `*` cannot be combined with concrete queue names - `*, mailers` will - just handle the `mailers` queue. + `*` cannot be combined with concrete queue names - `*, mailers` + just handles the `mailers` queue. When `sidekiq-cluster` is only running on a single node, make sure that at least - one process is running on all queues using `*`. This means a process will + one process is running on all queues using `*`. This means a process is This includes queues that have dedicated processes. If `sidekiq-cluster` is running on more than one node, you can also use @@ -116,83 +116,10 @@ you list: > - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. > - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. -In addition to selecting queues by name, as above, the `queue_selector` -option allows queue groups to be selected in a more general way using -the following components: - -- Attributes that can be selected. -- Operators used to construct a query. - -When `queue_selector` is set, all `queue_groups` must be in the queue -selector syntax. - -### Available attributes - -- [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1, `tags`. - -From the [list of all available -attributes](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml), -`queue_selector` allows selecting of queues by the following attributes: - -- `feature_category` - the [GitLab feature - category](https://about.gitlab.com/direction/maturity/#category-maturity) the - queue belongs to. For example, the `merge` queue belongs to the - `source_code_management` category. -- `has_external_dependencies` - whether or not the queue connects to external - services. For example, all importers have this set to `true`. -- `urgency` - how important it is that this queue's jobs run - quickly. Can be `high`, `low`, or `throttled`. For example, the - `authorized_projects` queue is used to refresh user permissions, and - is high urgency. -- `worker_name` - the worker name. The other attributes are typically more useful as - they are more general, but this is available in case a particular worker needs - to be selected. -- `name` - the queue name. Similiarly, this is available in case a particular queue needs - to be selected. -- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or - `unknown`. For example, the `project_export` queue is memory bound as it has - to load data in memory before saving it for export. -- `tags` - short-lived annotations for queues. These are expected to frequently - change from release to release, and may be removed entirely. - -`has_external_dependencies` is a boolean attribute: only the exact -string `true` is considered true, and everything else is considered -false. - -`tags` is a set, which means that `=` checks for intersecting sets, and -`!=` checks for disjoint sets. For example, `tags=a,b` selects queues -that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have -neither of those tags. - -### Available operators - -`queue_selector` supports the following operators, listed from highest -to lowest precedence: - -- `|` - the logical OR operator. For example, `query_a|query_b` (where `query_a` - and `query_b` are queries made up of the other operators here) will include - queues that match either query. -- `&` - the logical AND operator. For example, `query_a&query_b` (where - `query_a` and `query_b` are queries made up of the other operators here) will - only include queues that match both queries. -- `!=` - the NOT IN operator. For example, `feature_category!=issue_tracking` - excludes all queues from the `issue_tracking` feature category. -- `=` - the IN operator. For example, `resource_boundary=cpu` includes all - queues that are CPU bound. -- `,` - the concatenate set operator. For example, - `feature_category=continuous_integration,pages` includes all queues from - either the `continuous_integration` category or the `pages` category. This - example is also possible using the OR operator, but allows greater brevity, as - well as being lower precedence. - -The operator precedence for this syntax is fixed: it's not possible to make AND -have higher precedence than OR. - -[In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and -later, as with the standard queue group syntax above, a single `*` as the -entire queue group selects all queues. - -### Example queries +In addition to selecting queues by name, as above, the `queue_selector` option +allows queue groups to be selected in a more general way using a [worker matching +query](extra_sidekiq_routing.md#worker-matching-query). After `queue_selector` +is set, all `queue_groups` must follow the aforementioned syntax. In `/etc/gitlab/gitlab.rb`: @@ -215,7 +142,7 @@ WARNING: Sidekiq cluster is [scheduled](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240) to be the only way to start Sidekiq in GitLab 14.0. -By default, the Sidekiq service will run `sidekiq-cluster`. To disable this behavior, +By default, the Sidekiq service runs `sidekiq-cluster`. To disable this behavior, add the following to the Sidekiq configuration: ```ruby @@ -224,7 +151,7 @@ sidekiq['cluster'] = false ``` All of the aforementioned configuration options for `sidekiq` -are available. By default, they will be configured as follows: +are available. By default, they are configured as follows: ```ruby sidekiq['queue_selector'] = false @@ -241,14 +168,14 @@ cluster as above. When disabling `sidekiq_cluster`, you must copy your configuration for `sidekiq_cluster`over to `sidekiq`. Anything configured for -`sidekiq_cluster` will be overridden by the options for `sidekiq` when +`sidekiq_cluster` is overridden by the options for `sidekiq` when setting `sidekiq['cluster'] = true`. -When using this feature, the service called `sidekiq` will now be +When using this feature, the service called `sidekiq` is now running `sidekiq-cluster`. The [concurrency](#manage-concurrency) and other options configured -for Sidekiq will be respected. +for Sidekiq are respected. By default, logs for `sidekiq-cluster` go to `/var/log/gitlab/sidekiq` like regular Sidekiq logs. @@ -293,7 +220,7 @@ use all of its resources to perform those operations. To set up a separate Each process defined under `sidekiq` starts with a number of threads that equals the number of queues, plus one spare thread. For example, a process that handles the `process_commit` and `post_receive` -queues will use three threads in total. +queues uses three threads in total. ## Manage concurrency @@ -324,16 +251,16 @@ Running Sidekiq cluster is the default in GitLab 13.0 and later. ``` `min_concurrency` and `max_concurrency` are independent; one can be set without -the other. Setting `min_concurrency` to `0` will disable the limit. +the other. Setting `min_concurrency` to `0` disables the limit. For each queue group, let `N` be one more than the number of queues. The -concurrency factor will be set to: +concurrency factor are set to: 1. `N`, if it's between `min_concurrency` and `max_concurrency`. 1. `max_concurrency`, if `N` exceeds this value. 1. `min_concurrency`, if `N` is less than this value. -If `min_concurrency` is equal to `max_concurrency`, then this value will be used +If `min_concurrency` is equal to `max_concurrency`, then this value is used regardless of the number of queues. When `min_concurrency` is greater than `max_concurrency`, it is treated as @@ -360,7 +287,7 @@ Running Sidekiq directly is scheduled to be removed in GitLab sudo gitlab-ctl reconfigure ``` -This will set the concurrency (number of threads) for the Sidekiq process. +This sets the concurrency (number of threads) for the Sidekiq process. ## Modify the check interval @@ -426,21 +353,21 @@ you'd use the following: ### Monitor the `sidekiq-cluster` command -The `sidekiq-cluster` command will not terminate once it has started the desired -amount of Sidekiq processes. Instead, the process will continue running and +The `sidekiq-cluster` command does not terminate once it has started the desired +amount of Sidekiq processes. Instead, the process continues running and forward any signals to the child processes. This makes it easy to stop all Sidekiq processes as you simply send a signal to the `sidekiq-cluster` process, instead of having to send it to the individual processes. If the `sidekiq-cluster` process crashes or receives a `SIGKILL`, the child -processes will terminate themselves after a few seconds. This ensures you don't +processes terminate themselves after a few seconds. This ensures you don't end up with zombie Sidekiq processes. All of this makes monitoring the processes fairly easy. Simply hook up `sidekiq-cluster` to your supervisor of choice (for example, runit) and you're good to go. -If a child process died the `sidekiq-cluster` command will signal all remaining +If a child process died the `sidekiq-cluster` command signals all remaining process to terminate, then terminate itself. This removes the need for `sidekiq-cluster` to re-implement complex process monitoring/restarting code. Instead you should make sure your supervisor restarts the `sidekiq-cluster` @@ -456,7 +383,7 @@ file is written, but this can be changed by passing the `--pidfile` option to /opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit ``` -Keep in mind that the PID file will contain the PID of the `sidekiq-cluster` +Keep in mind that the PID file contains the PID of the `sidekiq-cluster` command and not the PID(s) of the started Sidekiq processes. ### Environment diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md new file mode 100644 index 00000000000..93cf8bd4f43 --- /dev/null +++ b/doc/administration/operations/extra_sidekiq_routing.md @@ -0,0 +1,164 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Queue routing rules **(FREE SELF)** + +When the number of Sidekiq jobs increases to a certain scale, the system faces +some scalability issues. One of them is that the length of the queue tends to get +longer. High-urgency jobs have to wait longer until other less urgent jobs +finish. This head-of-line blocking situation may eventually affect the +responsiveness of the system, especially critical actions. In another scenario, +the performance of some jobs is degraded due to other long running or CPU-intensive jobs +(computing or rendering ones) in the same machine. + +To counter the aforementioned issues, one effective solution is to split +Sidekiq jobs into different queues and assign machines handling each queue +exclusively. For example, all CPU-intensive jobs could be routed to the +`cpu-bound` queue and handled by a fleet of CPU optimized instances. The queue +topology differs between companies depending on the workloads and usage +patterns. Therefore, GitLab supports a flexible mechanism for the +administrator to route the jobs based on their characteristics. + +As an alternative to [Queue selector](extra_sidekiq_processes.md#queue-selector), which +configures Sidekiq cluster to listen to a specific set of workers or queues, +GitLab also supports routing a job from a worker to the desired queue when it +is scheduled. Sidekiq clients try to match a job against a configured list of +routing rules. Rules are evaluated from first to last, and as soon as we find a +match for a given worker we stop processing for that worker (first match wins). +If the worker doesn't match any rule, it falls back to the queue name generated +from the worker name. + +By default, if the routing rules are not configured (or denoted with an empty +array), all the jobs are routed to the queue generated from the worker name. + +## Example configuration + +In `/etc/gitlab/gitlab.rb`: + +```ruby +sidekiq['routing_rules'] = [ + # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue + ['resource_boundary!=cpu&urgency=high', 'high-urgency'], + # Route all database, gitaly and global search workers that are throttled to `throttled` queue + ['feature_category=database,gitaly,global_search&urgency=throttled', 'throttled'], + # Route all workers having contact with outside work to a `network-intenstive` queue + ['has_external_dependencies=true|feature_category=hooks|tags=network', 'network-intensive'], + # Route all import workers to the queues generated by the worker name, for + # example, JiraImportWorker to `jira_import`, SVNWorker to `svn_worker` + ['feature_category=import', nil], + # Wildcard matching, route the rest to `default` queue + ['*', 'default'] +] +``` + +The routing rules list is an order-matter array of tuples of query and +corresponding queue: + +- The query is following a [worker matching query](#worker-matching-query) syntax. +- The `` must be a valid Sidekiq queue name. If the queue name + is `nil`, or an empty string, the worker is routed to the queue generated + by the name of the worker instead. + +The query supports wildcard matching `*`, which matches all workers. As a +result, the wildcard query must stay at the end of the list or the rules after it +are ignored. + +NOTE: +Mixing queue routing rules and queue selectors requires care to +ensure all jobs that are scheduled and picked up by appropriate Sidekiq +workers. + +## Worker matching query + +GitLab provides a simple query syntax to match a worker based on its +attributes. This query syntax is employed by both [Queue routing +rules](#queue-routing-rules) and [Queue +selector](extra_sidekiq_processes.md#queue-selector). A query includes two +components: + +- Attributes that can be selected. +- Operators used to construct a query. + +### Available attributes + +> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). + +Queue matching query works upon the worker attributes, described in [Sidekiq +style guide](../../development/sidekiq_style_guide.md). We support querying +based on a subset of worker attributes: + +- `feature_category` - the [GitLab feature + category](https://about.gitlab.com/direction/maturity/#category-maturity) the + queue belongs to. For example, the `merge` queue belongs to the + `source_code_management` category. +- `has_external_dependencies` - whether or not the queue connects to external + services. For example, all importers have this set to `true`. +- `urgency` - how important it is that this queue's jobs run + quickly. Can be `high`, `low`, or `throttled`. For example, the + `authorized_projects` queue is used to refresh user permissions, and + is high urgency. +- `worker_name` - the worker name. The other attributes are typically more useful as + they are more general, but this is available in case a particular worker needs + to be selected. +- `name` - the queue name. The other attributes are typically more useful as + they are more general, but this is available in case a particular queue needs + to be selected. +- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or + `unknown`. For example, the `ProjectExportWorker` is memory bound as it has + to load data in memory before saving it for export. +- `tags` - short-lived annotations for queues. These are expected to frequently + change from release to release, and may be removed entirely. + +`has_external_dependencies` is a boolean attribute: only the exact +string `true` is considered true, and everything else is considered +false. + +`tags` is a set, which means that `=` checks for intersecting sets, and +`!=` checks for disjoint sets. For example, `tags=a,b` selects queues +that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have +neither of those tags. + +The attributes of each worker are hard-coded in the source code. For +convenience, we generate a [list of all available attributes in +GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) +and a [list of all available attributes in +GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml). + +### Available operators + +`queue_selector` supports the following operators, listed from highest +to lowest precedence: + +- `|` - the logical OR operator. For example, `query_a|query_b` (where `query_a` + and `query_b` are queries made up of the other operators here) will include + queues that match either query. +- `&` - the logical AND operator. For example, `query_a&query_b` (where + `query_a` and `query_b` are queries made up of the other operators here) will + only include queues that match both queries. +- `!=` - the NOT IN operator. For example, `feature_category!=issue_tracking` + excludes all queues from the `issue_tracking` feature category. +- `=` - the IN operator. For example, `resource_boundary=cpu` includes all + queues that are CPU bound. +- `,` - the concatenate set operator. For example, + `feature_category=continuous_integration,pages` includes all queues from + either the `continuous_integration` category or the `pages` category. This + example is also possible using the OR operator, but allows greater brevity, as + well as being lower precedence. + +The operator precedence for this syntax is fixed: it's not possible to make AND +have higher precedence than OR. + +[In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and +later, as with the standard queue group syntax above, a single `*` as the +entire queue group selects all queues. + +### Migration + +After the Sidekiq routing rules are changed, administrators need to take care +with the migration to avoid losing jobs entirely, especially in a system with +long queues of jobs. The migration can be done by following the migration steps +mentioned in [Sidekiq job +migration](../../raketasks/sidekiq_job_migration.md) diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 980db9713ee..8acc40da4ab 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Fast lookup of authorized SSH keys in the database +# Fast lookup of authorized SSH keys in the database **(FREE SELF)** NOTE: This document describes a drop-in replacement for the @@ -34,8 +34,15 @@ feature for CentOS 6, follow [the instructions on how to build and install a cus ## Fast lookup is required for Geo **(PREMIUM)** -By default, GitLab manages an `authorized_keys` file, which contains all the -public SSH keys for users allowed to access GitLab. However, to maintain a +By default, GitLab manages an `authorized_keys` file that is located in the +`git` user's home directory. For most installations, this will be located under +`/var/opt/gitlab/.ssh/authorized_keys`, but you can use the following command to locate the `authorized_keys` on your system.: + +```shell +getent passwd git | cut -d: -f6 | awk '{print $1"/.ssh/authorized_keys"}' +``` + +The `authorized_keys` file contains all the public SSH keys for users allowed to access GitLab. However, to maintain a single source of truth, [Geo](../geo/index.md) needs to be configured to perform SSH fingerprint lookups via database lookup. @@ -73,7 +80,7 @@ sudo service sshd reload ``` Confirm that SSH is working by commenting out your user's key in the `authorized_keys` -(start the line with a `#` to comment it), and attempting to pull a repository. +file (start the line with a `#` to comment it), and attempting to pull a repository. A successful pull would mean that GitLab was able to find the key in the database, since it is not present in the file anymore. @@ -219,5 +226,5 @@ the database. The following instructions can be used to build OpenSSH 7.5: GitLab supports `authorized_keys` database lookups with [SELinux](https://en.wikipedia.org/wiki/Security-Enhanced_Linux). Because the SELinux policy is static, GitLab doesn't support the ability to change -internal Unicorn ports at the moment. Administrators would have to create a special `.te` +internal webserver ports at the moment. Administrators would have to create a special `.te` file for the environment, since it isn't generated dynamically. diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index ffce104e1ad..a0ad2e24a4c 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# File system Performance Benchmarking +# File system performance benchmarking **(FREE SELF)** File system performance has a big impact on overall GitLab performance, especially for actions that read or write to Git repositories. This information diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 708861d8529..4b16c3b3a7e 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Performing Operations in GitLab +# Performing operations in GitLab **(FREE SELF)** Keep your GitLab instance up and running smoothly. @@ -21,8 +21,8 @@ Keep your GitLab instance up and running smoothly. - [Sidekiq MemoryKiller](sidekiq_memory_killer.md): Configure Sidekiq MemoryKiller to restart Sidekiq. - [Multiple Sidekiq processes](extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that need to be processed. **(FREE SELF)** +- [Sidekiq routing rules](extra_sidekiq_routing.md): Configure the routing rules to route a job from a worker to a desirable queue. **(FREE SELF)** - [Puma](puma.md): Understand Puma and puma-worker-killer. -- [Unicorn](unicorn.md): Understand Unicorn and unicorn-worker-killer. - Speed up SSH operations by [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or by [doing away with user SSH keys stored on GitLab entirely in favor diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index 3b676010bfe..fffff78b9d6 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -4,11 +4,10 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Switching to Puma +# Switching to Puma **(FREE SELF)** As of GitLab 12.9, [Puma](https://github.com/puma/puma) has replaced [Unicorn](https://yhbt.net/unicorn/) -as the default web server. From GitLab 13.0, the following run Puma instead of Unicorn unless -explicitly configured not to: +as the default web server. From GitLab 14.0, the following run Puma: - All-in-one package-based installations. - Helm chart-based installations. @@ -25,7 +24,7 @@ Multi-threaded Puma can therefore still serve more requests than a single proces ## Configuring Puma to replace Unicorn -Beginning with GitLab 13.0, Puma is the default application server. We plan to remove support for +Beginning with GitLab 13.0, Puma is the default application server. We removed support for Unicorn in GitLab 14.0. When switching to Puma, Unicorn server configuration diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index c7f00d05213..d3019e2c580 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -19,7 +19,7 @@ _only_ for Omnibus packages. The reason for this is that the MemoryKiller relies on runit to restart Sidekiq after a memory-induced shutdown and GitLab installations from source do not all use runit or an equivalent. -With the default settings, the MemoryKiller will cause a Sidekiq restart no +With the default settings, the MemoryKiller causes a Sidekiq restart no more often than once every 15 minutes, with the restart causing about one minute of delay for incoming background jobs. @@ -48,13 +48,13 @@ The MemoryKiller is controlled using environment variables. `SIDEKIQ_MEMORY_KILLER_MAX_RSS` defines the Sidekiq process allowed RSS. In _legacy_ mode, if the Sidekiq process exceeds the allowed RSS then an irreversible - delayed graceful restart will be triggered. The restart of Sidekiq will happen + delayed graceful restart is triggered. The restart of Sidekiq happens after `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` seconds. In _daemon_ mode, if the Sidekiq process exceeds the allowed RSS for longer than - `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` the graceful restart will be triggered. If the + `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` the graceful restart is triggered. If the Sidekiq process go below the allowed RSS within `SIDEKIQ_MEMORY_KILLER_GRACE_TIME`, - the restart will be aborted. + the restart is aborted. The default value for Omnibus packages is set [in the Omnibus GitLab @@ -71,13 +71,13 @@ The MemoryKiller is controlled using environment variables. The usage of this variable is described as part of `SIDEKIQ_MEMORY_KILLER_MAX_RSS`. - `SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT`: defaults to 30 seconds. This defines the - maximum time allowed for all Sidekiq jobs to finish. No new jobs will be accepted - during that time, and the process will exit as soon as all jobs finish. + maximum time allowed for all Sidekiq jobs to finish. No new jobs are accepted + during that time, and the process exits as soon as all jobs finish. - If jobs do not finish during that time, the MemoryKiller will interrupt all currently + If jobs do not finish during that time, the MemoryKiller interrupts all currently running jobs by sending `SIGTERM` to the Sidekiq process. If the process hard shutdown/restart is not performed by Sidekiq, - the Sidekiq process will be forcefully terminated after + the Sidekiq process is forcefully terminated after `Sidekiq.options[:timeout] + 2` seconds. An external supervision mechanism (e.g. runit) must restart Sidekiq afterwards. diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index cc09ad95dce..508d284b0bd 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# User lookup via OpenSSH's AuthorizedPrincipalsCommand +# User lookup via OpenSSH's AuthorizedPrincipalsCommand **(FREE SELF)** > [Available in](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19911) GitLab > Community Edition 11.2. diff --git a/doc/administration/operations/unicorn.md b/doc/administration/operations/unicorn.md index 03995ee05ba..6cee19186f9 100644 --- a/doc/administration/operations/unicorn.md +++ b/doc/administration/operations/unicorn.md @@ -1,115 +1,9 @@ --- -stage: Enablement -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'puma.md' +remove_date: '2021-08-26' --- -# Understanding Unicorn and unicorn-worker-killer +This file was moved to [another location](puma.md). -NOTE: -Starting with GitLab 13.0, Puma is the default web server used in GitLab -all-in-one package based installations as well as GitLab Helm chart deployments. - -## Unicorn - -GitLab uses [Unicorn](https://yhbt.net/unicorn/), a pre-forking Ruby web -server, to handle web requests (web browsers and Git HTTP clients). Unicorn is -a daemon written in Ruby and C that can load and run a Ruby on Rails -application; in our case the Rails application is GitLab Community Edition or -GitLab Enterprise Edition. - -Unicorn has a multi-process architecture to make better use of available CPU -cores (processes can run on different cores) and to have stronger fault -tolerance (most failures stay isolated in only one process and cannot take down -GitLab entirely). On startup, the Unicorn 'master' process loads a clean Ruby -environment with the GitLab application code, and then spawns 'workers' which -inherit this clean initial environment. The 'master' never handles any -requests, that is left to the workers. The operating system network stack -queues incoming requests and distributes them among the workers. - -In a perfect world, the master would spawn its pool of workers once, and then -the workers handle incoming web requests one after another until the end of -time. In reality, worker processes can crash or time out: if the master notices -that a worker takes too long to handle a request it will terminate the worker -process with SIGKILL ('kill -9'). No matter how the worker process ended, the -master process will replace it with a new 'clean' process again. Unicorn is -designed to be able to replace 'crashed' workers without dropping user -requests. - -This is what a Unicorn worker timeout looks like in `unicorn_stderr.log`. The -master process has PID 56227 below. - -```plaintext -[2015-06-05T10:58:08.660325 #56227] ERROR -- : worker=10 PID:53009 timeout (61s > 60s), killing -[2015-06-05T10:58:08.699360 #56227] ERROR -- : reaped # worker=10 -[2015-06-05T10:58:08.708141 #62538] INFO -- : worker=10 spawned pid=62538 -[2015-06-05T10:58:08.708824 #62538] INFO -- : worker=10 ready -``` - -### Tunable options - -The main tunable options for Unicorn are the number of worker processes and the -request timeout after which the Unicorn master terminates a worker process. -See the [Omnibus GitLab Unicorn settings -documentation](https://docs.gitlab.com/omnibus/settings/unicorn.html) -if you want to adjust these settings. - -## unicorn-worker-killer - -GitLab has memory leaks. These memory leaks manifest themselves in long-running -processes, such as Unicorn workers. (The Unicorn master process is not known to -leak memory, probably because it does not handle user requests.) - -To make these memory leaks manageable, GitLab comes with the -[unicorn-worker-killer gem](https://github.com/kzk/unicorn-worker-killer). This -gem [monkey-patches](https://en.wikipedia.org/wiki/Monkey_patch) the Unicorn -workers to do a memory self-check after every 16 requests. If the memory of the -Unicorn worker exceeds a pre-set limit then the worker process exits. The -Unicorn master then automatically replaces the worker process. - -This is a robust way to handle memory leaks: Unicorn is designed to handle -workers that 'crash' so no user requests will be dropped. The -unicorn-worker-killer gem is designed to only terminate a worker process _in -between requests_, so no user requests are affected. You can set the minimum and -maximum memory threshold (in bytes) for the Unicorn worker killer by -setting the following values `/etc/gitlab/gitlab.rb`: - -- For GitLab **12.7** and newer: - - ```ruby - unicorn['worker_memory_limit_min'] = "1024 * 1 << 20" - unicorn['worker_memory_limit_max'] = "1280 * 1 << 20" - ``` - -- For GitLab **12.6** and older: - - ```ruby - unicorn['worker_memory_limit_min'] = "400 * 1 << 20" - unicorn['worker_memory_limit_max'] = "650 * 1 << 20" - ``` - -Otherwise, you can set the `GITLAB_UNICORN_MEMORY_MIN` and `GITLAB_UNICORN_MEMORY_MAX` -[environment variables](../environment_variables.md). - -This is what a Unicorn worker memory restart looks like in unicorn_stderr.log. -You see that worker 4 (PID 125918) is inspecting itself and decides to exit. -The threshold memory value was 254802235 bytes, about 250MB. With GitLab this -threshold is a random value between 200 and 250 MB. The master process (PID -117565) then reaps the worker process and spawns a new 'worker 4' with PID -127549. - -```plaintext -[2015-06-05T12:07:41.828374 #125918] WARN -- : #: worker (pid: 125918) exceeds memory limit (256413696 bytes > 254802235 bytes) -[2015-06-05T12:07:41.828472 #125918] WARN -- : Unicorn::WorkerKiller send SIGQUIT (pid: 125918) alive: 23 sec (trial 1) -[2015-06-05T12:07:42.025916 #117565] INFO -- : reaped # worker=4 -[2015-06-05T12:07:42.034527 #127549] INFO -- : worker=4 spawned pid=127549 -[2015-06-05T12:07:42.035217 #127549] INFO -- : worker=4 ready -``` - -One other thing that stands out in the log snippet above, taken from -GitLab.com, is that 'worker 4' was serving requests for only 23 seconds. This -is a normal value for our current GitLab.com setup and traffic. - -The high frequency of Unicorn memory restarts on some GitLab sites can be a -source of confusion for administrators. Usually they are a [red -herring](https://en.wikipedia.org/wiki/Red_herring). + + diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 14e54513536..a6829b90f18 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -44,7 +44,7 @@ If you have installed GitLab from source: 1. After the installation is complete, to enable it, you must configure the Registry's settings in `gitlab.yml`. 1. Use the sample NGINX configuration file from under - [`lib/support/nginx/registry-ssl`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/nginx/registry-ssl) and edit it to match the + [`lib/support/nginx/registry-ssl`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/support/nginx/registry-ssl) and edit it to match the `host`, `port`, and TLS certificate paths. The contents of `gitlab.yml` are: @@ -417,8 +417,27 @@ To configure the `s3` storage driver in Omnibus: } ``` - - `regionendpoint` is only required when configuring an S3 compatible service such as MinIO. It takes a URL such as `http://127.0.0.1:9000`. + If using with an [AWS S3 VPC endpoint](https://docs.aws.amazon.com/vpc/latest/privatelink/vpc-endpoints-s3.html), + then set `regionendpoint` to your VPC endpoint address and set `path_style` to false: + + ```ruby + registry['storage'] = { + 's3' => { + 'accesskey' => 's3-access-key', + 'secretkey' => 's3-secret-key-for-access-key', + 'bucket' => 'your-s3-bucket', + 'region' => 'your-s3-region', + 'regionendpoint' => 'your-s3-vpc-endpoint', + 'path_style' => false + } + } + ``` + + - `regionendpoint` is only required when configuring an S3 compatible service such as MinIO, or + when using an AWS S3 VPC Endpoint. - `your-s3-bucket` should be the name of a bucket that exists, and can't include subdirectories. + - `path_style` should be set to true to use `host/bucket_name/object` style paths instead of + `bucket_name.host/object`. [Set to false for AWS S3](https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/). 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -1274,6 +1293,88 @@ curl "localhost:5001/debug/health" curl "localhost:5001/debug/vars" ``` +### Access old schema v1 Docker images + +Support for the [Docker registry v1 API](https://www.docker.com/blog/registry-v1-api-deprecation/), +including [schema V1 image manifests](https://docs.docker.com/registry/spec/manifest-v2-1/), +was: + +- [Deprecated in GitLab 13.7](https://about.gitlab.com/releases/2020/12/22/gitlab-13-7-released/#deprecate-pulls-that-use-v1-of-the-docker-registry-api) +- [Removed in GitLab 13.9](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#deprecate-pulls-that-use-v1-of-the-docker-registry-api) + +It's no longer possible to push or pull v1 images from the GitLab Container Registry. + +If you had v1 images in the GitLab Container Registry, but you did not upgrade them (following the +[steps Docker recommends](https://docs.docker.com/registry/spec/deprecated-schema-v1/)) +ahead of the GitLab 13.9 upgrade, these images are no longer accessible. If you try to pull them, +this error appears: + +- `Error response from daemon: manifest invalid: Schema 1 manifest not supported` + +For Self-Managed GitLab instances, you can regain access to these images by temporarily downgrading +the GitLab Container Registry to a version lower than `v3.0.0-gitlab`. Follow these steps to regain +access to these images: + +1. Downgrade the Container Registry to [`v2.13.1-gitlab`](https://gitlab.com/gitlab-org/container-registry/-/releases/v2.13.1-gitlab). +1. Upgrade any v1 images. +1. Revert the Container Registry downgrade. + +There's no need to put the registry in read-only mode during the image upgrade process. Ensure that +you are not relying on any new feature introduced since `v3.0.0-gitlab`. Such features are +unavailable during the upgrade process. See the [complete registry changelog](https://gitlab.com/gitlab-org/container-registry/-/blob/master/CHANGELOG.md) +for more information. + +The following sections provide additional details about each installation method. + +#### Helm chart installations + +For Helm chart installations: + +1. Override the [`image.tag`](https://docs.gitlab.com/charts/charts/registry/#configuration) + configuration parameter with `v2.13.1-gitlab`. +1. Restart. +1. Performing the [images upgrade](#images-upgrade)) steps. +1. Revert the `image.tag` parameter to the previous value. + +No other registry configuration changes are required. + +#### Omnibus installations + +For Omnibus installations: + +1. Temporarily replace the registry binary that ships with GitLab 13.9+ for one prior to + `v3.0.0-gitlab`. To do so, pull a previous version of the Docker image for the GitLab Container + Registry, such as `v2.13.1-gitlab`. You can then grab the `registry` binary from within this + image, located at `/bin/registry`: + + ```shell + id=$(docker create registry.gitlab.com/gitlab-org/build/cng/gitlab-container-registry:v2.13.1-gitlab) + docker cp $id:/bin/registry registry-2.13.1-gitlab + docker rm $id + ``` + +1. Replace the binary embedded in the Omnibus install, located at + `/opt/gitlab/embedded/bin/registry`, with `registry-2.13.1-gitlab`. Make sure to start by backing + up the original binary embedded in Omnibus, and restore it after performing the + [image upgrade](#images-upgrade)) steps. You should [stop](https://docs.gitlab.com/omnibus/maintenance/#starting-and-stopping) + the registry service before replacing its binary and start it right after. No registry + configuration changes are required. + +#### Source installations + +For source installations, locate your `registry` binary and temporarily replace it with the one +obtained from `v3.0.0-gitlab`, as explained for [Omnibus installations](#omnibus-installations). +Make sure to start by backing up the original registry binary, and restore it after performing the +[images upgrade](#images-upgrade)) +steps. + +#### Images upgrade + +Follow the [steps that Docker recommends to upgrade v1 images](https://docs.docker.com/registry/spec/deprecated-schema-v1/). +The most straightforward option is to pull those images and push them once again to the registry, +using a Docker client version above v1.12. Docker converts images automatically before pushing them +to the registry. Once done, all your v1 images should now be available as v2 images. + ### Advanced Troubleshooting We use a concrete example to illustrate how to diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index b454728cc8b..c4906ef6d8e 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -32,6 +32,23 @@ To enable the dependency proxy feature: 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab") for the changes to take effect. 1. Enable the [Puma web server](https://docs.gitlab.com/omnibus/settings/puma.html). +**Helm chart installations** + +1. After the installation is complete, update the global `appConfig` to enable the feature: + + ```yaml + global: + appConfig: + dependencyProxy: + enabled: true + bucket: gitlab-dependency-proxy + connection: {} + secret: + key: + ``` + +For more information, see [Configure Charts using Globals](https://docs.gitlab.com/charts/charts/globals.html#configure-appconfig-settings). + **Installations from source** 1. After the installation is complete, configure the `dependency_proxy` diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index c93ae90deb6..b9637f1b6f5 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -246,7 +246,7 @@ control over how the Pages daemon runs and serves content in your environment. | `gitlab_retrieval_timeout` | The maximum time to wait for a response from the GitLab API per request (default: 30s). | | `gitlab_retrieval_interval` | The interval to wait before retrying to resolve a domain's configuration via the GitLab API (default: 1s). | | `gitlab_retrieval_retries` | The maximum number of times to retry to resolve a domain's configuration via the API (default: 3). | -| `domain_config_source` | Domain configuration source (default: `auto`) | +| `domain_config_source` | This parameter was removed in 14.0, on earlier versions it can be used to enable and test API domain configuration source | | `gitlab_id` | The OAuth application public ID. Leave blank to automatically fill when Pages authenticates with GitLab. | | `gitlab_secret` | The OAuth application secret. Leave blank to automatically fill when Pages authenticates with GitLab. | | `auth_scope` | The OAuth application scope to use for authentication. Must match GitLab Pages OAuth application settings. Leave blank to use `api` scope by default. | @@ -281,6 +281,7 @@ control over how the Pages daemon runs and serves content in your environment. | **`pages_nginx[]`** | | | `enable` | Include a virtual host `server{}` block for Pages inside NGINX. Needed for NGINX to proxy traffic back to the Pages daemon. Set to `false` if the Pages daemon should directly receive all requests, for example, when using [custom domains](index.md#custom-domains). | | `FF_ENABLE_REDIRECTS` | Feature flag to disable redirects (enabled by default). Read the [redirects documentation](../../user/project/pages/redirects.md#disable-redirects) for more information. | +| `use_legacy_storage` | Temporarily-introduced parameter allowing to use legacy domain configuration source and storage. [Will be removed in GitLab 14.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6166). | --- @@ -756,51 +757,37 @@ Pages server. ## Domain source configuration -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217912) in GitLab 13.3. +When GitLab Pages daemon serves pages requests it firstly needs to identify which project should be used to +serve the requested URL and how its content is stored. -GitLab Pages can use different sources to get domain configuration. -The default value for Omnibus installations is `nil`. +Before GitLab 13.3, all pages content was extracted to the special shared directory, +and each project had a special configuration file. +The Pages daemon was reading these configuration files and storing their content in memory. - ```ruby - gitlab_pages['domain_config_source'] = nil - ``` +This approach had several disadvantages and was replaced with GitLab Pages using the internal GitLab API +every time a new domain is requested. +The domain information is also cached by the Pages daemon to speed up subsequent requests. -If left unchanged, GitLab Pages tries to use any available source (either `gitlab` or `disk`). The -preferred source is `gitlab`, which uses [API-based configuration](#gitlab-api-based-configuration). +From [GitLab 13.3 to GitLab 13.12](#domain-source-configuration-before-140) GitLab Pages supported both ways of obtaining domain information. -On large GitLab instances, using the API-based configuration significantly improves the pages daemon startup time, as there is no need to load all custom domains configuration into memory. +Starting from [GitLab 14.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993) GitLab Pages uses API +by default and fails to start if it can't connect to it. +For common issues, see the [troubleshooting section](#failed-to-connect-to-the-internal-gitlab-api). For more details see this [blog post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/). -### Deprecated `domain_config_source` - -WARNING: -The flag `gitlab_pages['domain_config_source']` is deprecated for use in [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/217913), -and is planned for removal in GitLab 14.0. - -GitLab 13.0 introduced the special flag `domain_config_source` to support manual opt-in to -[API-based configuration](#gitlab-api-based-configuration). -GitLab 13.7 introduced the [`auto` value](https://gitlab.com/gitlab-org/gitlab/-/issues/218358) -to support a smoother transition to API-based configuration. +### Domain source configuration before 14.0 -Starting with GitLab 14.0, GitLab Pages only supports API-based configuration, and -[disk source configuration is removed](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382). -Therefore, GitLab 14.0 also removes `domain_config_source`. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217912) in GitLab 13.3. -GitLab Pages fails to start if it can't connect to the GitLab API. For other common issues, see the -[troubleshooting section](#failed-to-connect-to-the-internal-gitlab-api) -or report an issue. +WARNING: +`domain_config_source` parameter is removed and has no effect starting from [GitLab 14.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993) -### GitLab API-based configuration +From [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/217912) to [GitLab 13.12](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993) GitLab Pages can either use `disk` or `gitlab` domain configuration source. -WARNING: -The flag `gitlab_pages['domain_config_source']` is deprecated for use in [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/217913), -and is planned for removal in GitLab 14.0. In GitLab 14.0 and later, GitLab Pages attempts to -connect to the API automatically, without requiring the manual configuration steps shown here. Pages -fails to start if this automatic connection fails. +We highly advise you to use `gitlab` configuration source as it will make transition to newer versions easier. -GitLab Pages can use an API-based configuration. This replaces disk source configuration, which -was used prior to GitLab 13.0. Follow these steps to enable it: +To explicitly enable API source: 1. Add the following to your `/etc/gitlab/gitlab.rb` file: @@ -810,14 +797,15 @@ was used prior to GitLab 13.0. Follow these steps to enable it: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -If you encounter an issue, you can disable it by choosing `disk`: +Or if you want to use legacy confiration source you can: -```ruby -gitlab_pages['domain_config_source'] = "disk" -``` +1. Add the following to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_pages['domain_config_source'] = "disk" + ``` -For other common issues, see the [troubleshooting section](#failed-to-connect-to-the-internal-gitlab-api) -or report an issue. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### GitLab API cache configuration @@ -831,8 +819,8 @@ or persistent errors, or the Pages Daemon serving old content. NOTE: Expiry, interval and timeout flags use [Golang's duration formatting](https://golang.org/pkg/time/#ParseDuration). A duration string is a possibly signed sequence of decimal numbers, -each with optional fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". -Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +each with optional fraction and a unit suffix, such as `300ms`, `1.5h` or `2h45m`. +Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. Examples: @@ -938,7 +926,7 @@ In installations from source: ## ZIP storage -In GitLab 14.0 the underlaying storage format of GitLab Pages is changing from +In GitLab 14.0 the underlying storage format of GitLab Pages is changing from files stored directly in disk to a single ZIP archive per project. These ZIP archives can be stored either locally on disk storage or on the [object storage](#using-object-storage) if it is configured. @@ -1052,11 +1040,12 @@ To migrate GitLab Pages to GitLab 14.0: 1. If your current GitLab version is lower than 13.12, then you first need to upgrade to 13.12. Upgrading directly to 14.0 may cause downtime for some web-sites hosted on GitLab Pages until you finish the following steps. -1. Enable the [API-based configuration](#gitlab-api-based-configuration), which +1. Set [`domain_config_source` to `gitlab`](#domain-source-configuration-before-140), which is the default starting from GitLab 14.0. Skip this step if you're already running GitLab 14.0 or above. 1. If you want to store your pages content in the [object storage](#using-object-storage), make sure to configure it. If you want to store the pages content locally or continue using an NFS server, skip this step. 1. [Migrate legacy storage to ZIP storage.](#migrate-legacy-storage-to-zip-storage) +1. Upgrade GitLab to 14.0. ## Backup @@ -1081,6 +1070,16 @@ but commented out to help encourage others to add to it in the future. --> ## Troubleshooting +### How to see GitLab Pages logs + +You can see Pages daemon logs by running: + +```shell +sudo gitlab-ctl tail gitlab-pages +``` + +You can also find the log file in `/var/log/gitlab/gitlab-pages/current`. + ### `open /etc/ssl/ca-bundle.pem: permission denied` GitLab Pages runs inside a `chroot` jail, usually in a uniquely numbered directory like @@ -1210,12 +1209,12 @@ These are due to the Pages files not being among the It is possible to copy the subfolders and files in the [Pages path](#change-storage-path) to the new primary node to resolve this. For example, you can adapt the `rsync` strategy from the -[moving repositories documenation](../operations/moving_repositories.md). +[moving repositories documentation](../operations/moving_repositories.md). Alternatively, run the CI pipelines of those projects that contain a `pages` job again. ### Failed to connect to the internal GitLab API -If you have enabled [API-based configuration](#gitlab-api-based-configuration) and see the following error: +If you see the following error: ```plaintext ERRO[0010] Failed to connect to the internal GitLab API after 0.50s error="failed to connect to internal Pages API: HTTP status: 401" @@ -1236,11 +1235,6 @@ error="failed to connect to internal Pages API: Get \"https://gitlab.example.com ### Pages cannot communicate with an instance of the GitLab API -WARNING: -The flag `gitlab_pages['domain_config_source']` is [deprecated](#deprecated-domain_config_source) -for use in [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/217913), -and is planned for removal in GitLab 14.0. - If you use the default value for `domain_config_source=auto` and run multiple instances of GitLab Pages, you may see intermittent 502 error responses while serving Pages content. You may also see the following warning in the Pages logs: @@ -1321,3 +1315,25 @@ To enable disk access: ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +### GitLab Pages doesn't work after upgrading to GitLab 14.0 or above + +GitLab 14.0 introduces a number of changes to GitLab Pages which may require manual intervention. + +1. Firstly [follow the migration guide](#migrate-gitlab-pages-to-140). +1. If it doesn't work, see [GitLab Pages logs](#how-to-see-gitlab-pages-logs), and if you see any errors there then search them on this page. + +WARNING: +As the last resort you can temporarily enable legacy storage and configuration mechanisms. Support for them [will be removed in GitLab 14.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6166), so GitLab Pages will stop working if don't resolve the underlying issue. + +To do that: + +1. Please describe the issue you're seeing in [here](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['use_legacy_storage'] = true + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/polling.md b/doc/administration/polling.md index f66df70a163..f6732b8edc6 100644 --- a/doc/administration/polling.md +++ b/doc/administration/polling.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Polling configuration +# Polling configuration **(FREE SELF)** The GitLab UI polls for updates for different resources (issue notes, issue titles, pipeline statuses, etc.) on a schedule appropriate to the resource. diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md index a9d0af952a0..8f0fe0ace87 100644 --- a/doc/administration/postgresql/external.md +++ b/doc/administration/postgresql/external.md @@ -22,6 +22,10 @@ If you use a cloud-managed service, or provide your own PostgreSQL instance: roles to your `gitlab` user: - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. - Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. + - Google Cloud SQL requires the [`cloudsqlsuperuser`](https://cloud.google.com/sql/docs/postgres/users#default-users) role. + + This is for the installation of extensions during installation and upgrades. As an alternative, + [ensure the extensions are installed manually, and read about the problems that may arise during future GitLab upgrades](../../install/postgresql_extensions.md). 1. Configure the GitLab application servers with the appropriate connection details for your external PostgreSQL service in your `/etc/gitlab/gitlab.rb` file: diff --git a/doc/administration/postgresql/img/pg_ha_architecture.png b/doc/administration/postgresql/img/pg_ha_architecture.png index ef870f652ae..5d2a4a584bf 100644 Binary files a/doc/administration/postgresql/img/pg_ha_architecture.png and b/doc/administration/postgresql/img/pg_ha_architecture.png differ diff --git a/doc/administration/postgresql/index.md b/doc/administration/postgresql/index.md index eabb396aeab..bce78bbccff 100644 --- a/doc/administration/postgresql/index.md +++ b/doc/administration/postgresql/index.md @@ -16,7 +16,7 @@ There are essentially three setups to choose from. This setup is for when you have installed GitLab using the [Omnibus GitLab **Enterprise Edition** (EE) package](https://about.gitlab.com/install/?version=ee). -All the tools that are needed like PostgreSQL, PgBouncer, Patroni, and repmgr are bundled in +All the tools that are needed like PostgreSQL, PgBouncer, and Patroni are bundled in the package, so you can it to set up the whole PostgreSQL infrastructure (primary, replica). [> Read how to set up PostgreSQL replication and failover using Omnibus GitLab](replication_and_failover.md) diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index fc7da04b960..e481fcb71f4 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -164,7 +164,7 @@ and [GitLab upgrades](https://docs.gitlab.com/omnibus/update/README.html#use-pos 1. To find the primary node, run the following on a database node: ```shell - sudo gitlab-ctl repmgr cluster show + sudo gitlab-ctl patroni members ``` 1. Edit `/etc/gitlab/gitlab.rb` on the application node you're performing the task on, and update diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index 878d2b536cb..b6d2e36851d 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -36,8 +36,8 @@ to avoid the network becoming a single point of failure. NOTE: As of GitLab 13.3, PostgreSQL 12 is shipped with Omnibus GitLab. Clustering for PostgreSQL 12 is only supported with -Patroni. See the [Patroni](#patroni) section for further details. The support for repmgr will not be extended beyond -PostgreSQL 11. +Patroni. See the [Patroni](#patroni) section for further details. Starting with GitLab 14.0, only PostgreSQL 12 is +shipped with Omnibus GitLab and thus Patroni becomes mandatory for replication and failover. ### Database node @@ -118,26 +118,27 @@ When using default setup, minimum configuration requires: Few notes on the service itself: - The service runs under a system account, by default `gitlab-consul`. - - If you are using a different username, you will have to specify it. We - will refer to it with `CONSUL_USERNAME`, -- There will be a database user created with read-only access to the repmgr - database -- Passwords will be stored in the following locations: + - If you are using a different username, you have to specify it through the + `CONSUL_USERNAME` variable. +- Passwords are stored in the following locations: - `/etc/gitlab/gitlab.rb`: hashed - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed - `/var/opt/gitlab/consul/.pgpass`: plaintext #### PostgreSQL information -When configuring PostgreSQL, we will set `max_wal_senders` to one more than -the number of database nodes in the cluster. -This is used to prevent replication from using up all of the -available database connections. +When configuring PostgreSQL, we do the following: + +- Set `max_replication_slots` to double the number of database nodes. + Patroni uses one extra slot per node when initiating the replication. +- Set `max_wal_senders` to one more than the allocated number of replication slots in the cluster. + This prevents replication from using up all of the available database connections. In this document we are assuming 3 database nodes, which makes this configuration: ```ruby -patroni['postgresql']['max_wal_senders'] = 4 +patroni['postgresql']['max_replication_slots'] = 6 +patroni['postgresql']['max_wal_senders'] = 7 ``` As previously mentioned, you'll have to prepare the network subnets that will @@ -176,9 +177,7 @@ Few notes on the service itself: - The service runs as the same system account as the database - In the package, this is by default `gitlab-psql` - If you use a non-default user account for PgBouncer service (by default `pgbouncer`), you will have to specify this username. We will refer to this requirement with `PGBOUNCER_USERNAME`. -- The service will have a regular database user account generated for it - - This defaults to `repmgr` -- Passwords will be stored in the following locations: +- Passwords are stored in the following locations: - `/etc/gitlab/gitlab.rb`: hashed, and in plain text - `/var/opt/gitlab/pgbouncer/pg_auth`: hashed @@ -198,8 +197,7 @@ When installing the GitLab package, do not supply `EXTERNAL_URL` value. #### Configuring Patroni cluster -You must enable Patroni explicitly to be able to use it (with `patroni['enable'] = true`). When Patroni is enabled -repmgr will be disabled automatically. +You must enable Patroni explicitly to be able to use it (with `patroni['enable'] = true`). Any PostgreSQL configuration item that controls replication, for example `wal_level`, `max_wal_senders`, etc, are strictly controlled by Patroni and will override the original settings that you make with the `postgresql[...]` configuration key. @@ -210,17 +208,14 @@ configuration key. NOTE: The configuration of a Patroni node is very similar to a repmgr but shorter. When Patroni is enabled, first you can ignore -any replication setting of PostgreSQL (it will be overwritten anyway). Then you can remove any `repmgr[...]` or +any replication setting of PostgreSQL (it is overwritten anyway). Then you can remove any `repmgr[...]` or repmgr-specific configuration as well. Especially, make sure that you remove `postgresql['shared_preload_libraries'] = 'repmgr_funcs'`. -Here is an example similar to [the one that was done with repmgr](#configuring-repmgr-nodes): +Here is an example: ```ruby -# Disable all components except PostgreSQL, Patroni (or Repmgr), and Consul -roles['postgres_role'] - -# Enable Patroni (which automatically disables Repmgr). -patroni['enable'] = true +# Disable all components except Patroni and Consul +roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -236,13 +231,20 @@ consul['services'] = %w(postgresql) # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' +# Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value +postgresql['sql_replication_password'] = 'POSTGRESQL_REPLICATION_PASSWORD_HASH' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' -# Replace X with value of number of db nodes + 1 (OPTIONAL the default value is 5) -patroni['postgresql']['max_wal_senders'] = X +# Sets `max_replication_slots` to double the number of database nodes. +# Patroni uses one extra slot per node when initiating the replication. patroni['postgresql']['max_replication_slots'] = X +# Set `max_wal_senders` to one more than the number of replication slots in the cluster. +# This is used to prevent replication from using up all of the +# available database connections. +patroni['postgresql']['max_wal_senders'] = X+1 + # Replace XXX.XXX.XXX.XXX/YY with Network Address postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) @@ -267,10 +269,6 @@ Generally, when Consul cluster is ready, the first node that [reconfigures](../r becomes the leader. You do not need to sequence the nodes reconfiguration. You can run them in parallel or in any order. If you choose an arbitrary order you do not have any predetermined master. -NOTE: -As opposed to repmgr, once the nodes are reconfigured you do not need any further action or additional command to join -the replicas. - #### Enable Monitoring > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3786) in GitLab 12.0. @@ -298,7 +296,7 @@ If you enable Monitoring, it must be enabled on **all** database servers. ```ruby # Disable all components except PgBouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -348,7 +346,7 @@ If you enable Monitoring, it must be enabled on **all** database servers. 1. Ensure each node is talking to the current master: ```shell - gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD + gitlab-ctl pgb-console # Supply PGBOUNCER_PASSWORD when prompted ``` If there is an error `psql: ERROR: Auth failed` after typing in the @@ -415,7 +413,7 @@ Refer to your preferred Load Balancer's documentation for further guidance. ### Configuring the Application nodes -These will be the nodes running the `gitlab-rails` service. You may have other +Application nodes run the `gitlab-rails` service. You may have other attributes set, but the following need to be set. 1. Edit `/etc/gitlab/gitlab.rb`: @@ -448,7 +446,7 @@ in the Troubleshooting section before proceeding. ### Backups -Do not backup or restore GitLab through a PgBouncer connection: this will cause a GitLab outage. +Do not backup or restore GitLab through a PgBouncer connection: this causes a GitLab outage. [Read more about this and how to reconfigure backups](../../raketasks/backup_restore.md#backup-and-restore-for-installations-using-pgbouncer). @@ -495,7 +493,7 @@ On each server edit `/etc/gitlab/gitlab.rb`: ```ruby # Disable all components except Consul -roles ['consul_role'] +roles(['consul_role']) consul['configuration'] = { server: true, @@ -512,7 +510,7 @@ On each server edit `/etc/gitlab/gitlab.rb`: ```ruby # Disable all components except Pgbouncer and Consul agent -roles ['pgbouncer_role'] +roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -527,7 +525,6 @@ pgbouncer['users'] = { } consul['watchers'] = %w(postgresql) -consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -545,29 +542,26 @@ An internal load balancer (TCP) is then required to be setup to serve each PgBou On database nodes edit `/etc/gitlab/gitlab.rb`: ```ruby -# Disable all components except PostgreSQL, Patroni (or Repmgr), and Consul -roles ['postgres_role'] +# Disable all components except Patroni and Consul +roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' -# Enable Patroni (which automatically disables Repmgr). -patroni['enable'] = true - # Disable automatic database migrations gitlab_rails['auto_migrate'] = false postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' -patroni['postgresql']['max_wal_senders'] = 4 +patroni['postgresql']['max_replication_slots'] = 6 +patroni['postgresql']['max_wal_senders'] = 7 postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) # Configure the Consul agent consul['services'] = %w(postgresql) -consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -586,19 +580,6 @@ After deploying the configuration follow these steps: gitlab-ctl get-postgresql-primary ``` -1. On the primary database node: - - Enable the `pg_trgm` and `btree_gist` extensions: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - - ```shell - CREATE EXTENSION pg_trgm; - CREATE EXTENSION btree_gist; - ``` - 1. On `10.6.0.41`, our application server: Set `gitlab-consul` user's PgBouncer password to `toomanysecrets`: @@ -640,17 +621,14 @@ Please note that after the initial configuration, if a failover occurs, the Post On database nodes edit `/etc/gitlab/gitlab.rb`: ```ruby -# Disable all components except PostgreSQL, Repmgr, and Consul -roles ['postgres_role'] +# Disable all components except Patroni and Consul +roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' postgresql['hot_standby'] = 'on' postgresql['wal_level'] = 'replica' -# Enable Patroni (which automatically disables Repmgr). -patroni['enable'] = true - # Disable automatic database migrations gitlab_rails['auto_migrate'] = false @@ -659,7 +637,15 @@ consul['services'] = %w(postgresql) postgresql['pgbouncer_user_password'] = '771a8625958a529132abe6f1a4acb19c' postgresql['sql_user_password'] = '450409b85a0223a214b5fb1484f34d0f' -patroni['postgresql']['max_wal_senders'] = 4 + +# Sets `max_replication_slots` to double the number of database nodes. +# Patroni uses one extra slot per node when initiating the replication. +patroni['postgresql']['max_replication_slots'] = 6 + +# Set `max_wal_senders` to one more than the number of replication slots in the cluster. +# This is used to prevent replication from using up all of the +# available database connections. +patroni['postgresql']['max_wal_senders'] = 7 postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/16) @@ -716,22 +702,17 @@ The manual steps for this configuration are the same as for the [example recomme ## Patroni NOTE: -Using Patroni instead of Repmgr is supported for PostgreSQL 11 and required for PostgreSQL 12. +Using Patroni instead of Repmgr is supported for PostgreSQL 11 and required for PostgreSQL 12. Starting with GitLab 14.0, only PostgreSQL 12 is available and hence Patroni is mandatory to achieve failover and replication. Patroni is an opinionated solution for PostgreSQL high-availability. It takes the control of PostgreSQL, overrides its -configuration and manages its lifecycle (start, stop, restart). This is a more active approach when compared to repmgr. -Both repmgr and Patroni are both supported and available. But Patroni will be the default (and perhaps the only) option -for PostgreSQL 12 clustering and cascading replication for Geo deployments. +configuration and manages its lifecycle (start, stop, restart). Patroni is the only option for PostgreSQL 12 clustering and for cascading replication for Geo deployments. The [architecture](#example-recommended-setup-manual-steps) (that was mentioned above) does not change for Patroni. You do not need any special consideration for Patroni while provisioning your database nodes. Patroni heavily relies on Consul to store the state of the cluster and elect a leader. Any failure in Consul cluster and its leader election will propagate to Patroni cluster as well. -Similar to repmgr, Patroni monitors the cluster and handles failover. When the primary node fails it works with Consul -to notify PgBouncer. However, as opposed to repmgr, on failure, Patroni handles the transitioning of the old primary to -a replica and rejoins it to the cluster automatically. So you do not need any manual operation for recovering the -cluster as you do with repmgr. +Patroni monitors the cluster and handles failover. When the primary node fails it works with Consul to notify PgBouncer. On failure, Patroni handles the transitioning of the old primary to a replica and rejoins it to the cluster automatically. With Patroni the connection flow is slightly different. Patroni on each node connects to Consul agent to join the cluster. Only after this point it decides if the node is the primary or a replica. Based on this decision, it configures @@ -829,7 +810,7 @@ For further details on this subject, see the #### Geo secondary site considerations -Similar to `repmgr`, when a Geo secondary site is replicating from a primary site that uses `Patroni` and `PgBouncer`, [replicating through PgBouncer is not supported](https://github.com/pgbouncer/pgbouncer/issues/382#issuecomment-517911529) and the secondary must replicate directly from the leader node in the `Patroni` cluster. Therefore, when there is an automatic or manual failover in the `Patroni` cluster, you will need to manually re-point your secondary site to replicate from the new leader with: +When a Geo secondary site is replicating from a primary site that uses `Patroni` and `PgBouncer`, [replicating through PgBouncer is not supported](https://github.com/pgbouncer/pgbouncer/issues/382#issuecomment-517911529) and the secondary must replicate directly from the leader node in the `Patroni` cluster. Therefore, when there is an automatic or manual failover in the `Patroni` cluster, you will need to manually re-point your secondary site to replicate from the new leader with: ```shell sudo gitlab-ctl replicate-geo-database --host= --replication-slot= @@ -891,8 +872,8 @@ You can switch an exiting database cluster to use Patroni instead of repmgr with 1. On the primary node, [configure Patroni](#configuring-patroni-cluster). Remove `repmgr` and any other repmgr-specific configuration. Also remove any configuration that is related to PostgreSQL replication. -1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on the primary node. It will become - the leader. You can check this with: +1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) on the primary node. + It makes it the leader. You can check this with: ```shell sudo gitlab-ctl tail patroni @@ -920,13 +901,13 @@ upgrading PostgreSQL. Here are a few key facts that you must consider before upgrading PostgreSQL: - The main point is that you will have to **shut down the Patroni cluster**. This means that your - GitLab deployment will be down for the duration of database upgrade or, at least, as long as your leader + GitLab deployment is down for the duration of database upgrade or, at least, as long as your leader node is upgraded. This can be **a significant downtime depending on the size of your database**. - Upgrading PostgreSQL creates a new data directory with a new control data. From Patroni's perspective this is a new cluster that needs to be bootstrapped again. Therefore, as part of the upgrade procedure, - the cluster state, which is stored in Consul, will be wiped out. Once the upgrade is completed, Patroni - will be instructed to bootstrap a new cluster. **Note that this will change your _cluster ID_**. + the cluster state (stored in Consul) is wiped out. Once the upgrade is completed, Patroni + bootstraps a new cluster. **Note that this changes your _cluster ID_**. - The procedures for upgrading leader and replicas are not the same. That is why it is important to use the right procedure on each node. @@ -996,389 +977,6 @@ Reverting PostgreSQL upgrade with `gitlab-ctl revert-pg-upgrade` has the same co `gitlab-ctl pg-upgrade`. You should follow the same procedure by first stopping the replicas, then reverting the leader, and finally reverting the replicas. -## Repmgr - -NOTE: -Using Patroni instead of Repmgr is supported for PostgreSQL 11 and required for PostgreSQL 12. - -### Configuring Repmgr Nodes - -1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - - ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the Consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - # Replace X with value of number of db nodes + 1 - postgresql['max_wal_senders'] = X - postgresql['max_replication_slots'] = X - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) - - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - > `postgres_role` was introduced with GitLab 10.3 - -1. On secondary nodes, add all the configuration specified above for primary node - to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration - to inform `gitlab-ctl` that they are standby nodes initially and it need not - attempt to register them as primary node - - ```ruby - # Specify if a node should attempt to be master on initialization - repmgr['master_on_initialization'] = false - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. [Enable Monitoring](#enable-monitoring) - -> Please note: -> -> - If you want your database to listen on a specific interface, change the configuration: -> `postgresql['listen_address'] = '0.0.0.0'`. -> - If your PgBouncer service runs under a different user account, -> you also need to specify: `postgresql['pgbouncer_user'] = PGBOUNCER_USERNAME` in -> your configuration. - -#### Database nodes post-configuration - -##### Primary node - -Select one node as a primary node. - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Enable the `pg_trgm` extension: - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Enable the `btree_gist` extension: - - ```shell - CREATE EXTENSION btree_gist; - ``` - -1. Exit the database prompt by typing `\q` and Enter. - -1. Verify the cluster is initialized with one node: - - ```shell - gitlab-ctl repmgr cluster show - ``` - - The output should be similar to the following: - - ```plaintext - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -1. Note down the hostname or IP address in the connection string: `host=HOSTNAME`. We will - refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value - is not an IP address, it will need to be a resolvable name (via DNS or - `/etc/hosts`) - -##### Secondary nodes - -1. Set up the repmgr standby: - - ```shell - gitlab-ctl repmgr standby setup MASTER_NODE_NAME - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - # gitlab-ctl repmgr standby setup MASTER_NODE_NAME - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` - -1. Verify the node now appears in the cluster: - - ```shell - gitlab-ctl repmgr cluster show - ``` - - The output should be similar to the following: - - ```plaintext - Role | Name | Upstream | Connection String - ----------+---------|-----------|------------------------------------------------ - * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` - -Repeat the above steps on all secondary nodes. - -#### Database checkpoint - -Before moving on, make sure the databases are configured correctly. Run the -following command on the **primary** node to verify that replication is working -properly: - -```shell -gitlab-ctl repmgr cluster show -``` - -The output should be similar to: - -```plaintext -Role | Name | Upstream | Connection String -----------+--------------|--------------|-------------------------------------------------------------------- -* master | MASTER | | host=MASTER port=5432 user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY port=5432 user=gitlab_repmgr dbname=gitlab_repmgr -``` - -If the 'Role' column for any node says "FAILED", check the -[Troubleshooting section](#troubleshooting) before proceeding. - -Also, check that the check master command works successfully on each node: - -```shell -su - gitlab-consul -gitlab-ctl repmgr-check-master || echo 'This node is a standby repmgr node' -``` - -This command relies on exit codes to tell Consul whether a particular node is a master -or secondary. The most important thing here is that this command does not produce errors. -If there are errors it's most likely due to incorrect `gitlab-consul` database user permissions. -Check the [Troubleshooting section](#troubleshooting) before proceeding. - -### Repmgr failover procedure - -By default, if the master database fails, `repmgrd` should promote one of the -standby nodes to master automatically, and Consul will update PgBouncer with -the new master. - -If you need to failover manually, you have two options: - -**Shutdown the current master database** - -Run: - -```shell -gitlab-ctl stop postgresql -``` - -The automated failover process will see this and failover to one of the -standby nodes. - -**Or perform a manual failover** - -1. Ensure the old master node is not still active. -1. Login to the server that should become the new master and run: - - ```shell - gitlab-ctl repmgr standby promote - ``` - -1. If there are any other standby servers in the cluster, have them follow - the new master server: - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - ``` - -#### Geo secondary site considerations - -When a Geo secondary site is replicating from a primary site that uses `repmgr` and `PgBouncer`, [replicating through PgBouncer is not supported](https://github.com/pgbouncer/pgbouncer/issues/382#issuecomment-517911529) and the secondary must replicate directly from the leader node in the `repmgr` cluster. Therefore, when there is a failover in the `repmgr` cluster, you will need to manually re-point your secondary site to replicate from the new leader with: - -```shell -sudo gitlab-ctl replicate-geo-database --host= --replication-slot= -``` - -Otherwise, the replication will not happen anymore, even if the original node gets re-added as a follower node. This will re-sync your secondary site database and may take a long time depending on the amount of data to sync. You may also need to run `gitlab-ctl reconfigure` if replication is still not working after re-syncing. - -### Repmgr Restore procedure - -If a node fails, it can be removed from the cluster, or added back as a standby -after it has been restored to service. - -#### Remove a standby from the cluster - - From any other node in the cluster, run: - - ```shell - gitlab-ctl repmgr standby unregister --node=X - ``` - - where X is the value of node in `repmgr.conf` on the old server. - - To find this, you can use: - - ```shell - awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf - ``` - - It will output something like: - - ```plaintext - 959789412 - ``` - - Then you will use this ID to unregister the node: - - ```shell - gitlab-ctl repmgr standby unregister --node=959789412 - ``` - -#### Add a node as a standby server - - From the standby node, run: - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - gitlab-ctl restart repmgrd - ``` - - WARNING: - When the server is brought back online, and before - you switch it to a standby node, repmgr will report that there are two masters. - If there are any clients that are still attempting to write to the old master, - this will cause a split, and the old master will need to be resynced from - scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. - -#### Add a failed master back into the cluster as a standby node - - Once `repmgrd` and PostgreSQL are running, the node will need to follow the new - as a standby node. - - ```shell - gitlab-ctl repmgr standby follow NEW_MASTER - ``` - - Once the node is following the new master as a standby, the node needs to be - [unregistered from the cluster on the new master node](#remove-a-standby-from-the-cluster). - - Once the old master node has been unregistered from the cluster, it will need - to be setup as a new standby: - - ```shell - gitlab-ctl repmgr standby setup NEW_MASTER - ``` - - Failure to unregister and read the old master node can lead to subsequent failovers - not working. - -### Alternate configurations - -#### Database authorization - -By default, we give any host on the database network the permission to perform -repmgr operations using PostgreSQL's `trust` method. If you do not want this -level of trust, there are alternatives. - -You can trust only the specific nodes that will be database clusters, or you -can require md5 authentication. - -#### Trust specific addresses - -If you know the IP address, or FQDN of all database and PgBouncer nodes in the -cluster, you can trust only those nodes. - -In `/etc/gitlab/gitlab.rb` on all of the database nodes, set -`repmgr['trust_auth_cidr_addresses']` to an array of strings containing all of -the addresses. - -If setting to a node's FQDN, they must have a corresponding PTR record in DNS. -If setting to a node's IP address, specify it as `XXX.XXX.XXX.XXX/32`. - -For example: - -```ruby -repmgr['trust_auth_cidr_addresses'] = %w(192.168.1.44/32 db2.example.com) -``` - -#### MD5 Authentication - -If you are running on an untrusted network, repmgr can use md5 authentication -with a [`.pgpass` file](https://www.postgresql.org/docs/11/libpq-pgpass.html) -to authenticate. - -You can specify by IP address, FQDN, or by subnet, using the same format as in -the previous section: - -1. On the current master node, create a password for the `gitlab` and - `gitlab_repmgr` user: - - ```shell - gitlab-psql -d template1 - template1=# \password gitlab_repmgr - Enter password: **** - Confirm password: **** - template1=# \password gitlab - ``` - -1. On each database node: - - 1. Edit `/etc/gitlab/gitlab.rb`: - 1. Ensure `repmgr['trust_auth_cidr_addresses']` is **not** set - 1. Set `postgresql['md5_auth_cidr_addresses']` to the desired value - 1. Set `postgresql['sql_replication_user'] = 'gitlab_repmgr'` - 1. Reconfigure with `gitlab-ctl reconfigure` - 1. Restart PostgreSQL with `gitlab-ctl restart postgresql` - - 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to - when asked: - - ```shell - gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' - ``` - -1. On each PgBouncer node, edit `/etc/gitlab/gitlab.rb`: - 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for - the `gitlab` database user - 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect - ## Troubleshooting ### Consul and PostgreSQL changes not taking effect @@ -1387,25 +985,10 @@ Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and P To restart either service, run `gitlab-ctl restart SERVICE` -For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. To be safe, you can stop `repmgrd` on the standby nodes first with `gitlab-ctl stop repmgrd`, then start afterwards with `gitlab-ctl start repmgrd`. +For PostgreSQL, it is usually safe to restart the master node by default. Automatic failover defaults to a 1 minute timeout. Provided the database returns before then, nothing else needs to be done. On the Consul server nodes, it is important to [restart the Consul service](../consul.md#restart-consul) in a controlled manner. -### `gitlab-ctl repmgr-check-master` command produces errors - -If this command displays errors about database permissions it is likely that something failed during -install, resulting in the `gitlab-consul` database user getting incorrect permissions. Follow these -steps to fix the problem: - -1. On the master database node, connect to the database prompt - `gitlab-psql -d template1` -1. Delete the `gitlab-consul` user - `DROP USER "gitlab-consul";` -1. Exit the database prompt - `\q` -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) and the user will be re-added with the proper permissions. -1. Change to the `gitlab-consul` user - `su - gitlab-consul` -1. Try the check command again - `gitlab-ctl repmgr-check-master`. - -Now there should not be errors. If errors still occur then there is another problem. - ### PgBouncer error `ERROR: pgbouncer cannot connect to server` You may get this error when running `gitlab-rake gitlab:db:configure` or you diff --git a/doc/administration/postgresql/standalone.md b/doc/administration/postgresql/standalone.md index cca46a2ea8c..b21625acb56 100644 --- a/doc/administration/postgresql/standalone.md +++ b/doc/administration/postgresql/standalone.md @@ -17,8 +17,8 @@ together with Omnibus GitLab. This is recommended as part of our 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using *steps 1 and 2* from the GitLab downloads page. - Do not complete any other steps on the download page. -1. Generate a password hash for PostgreSQL. This assumes you will use the default - username of `gitlab` (recommended). The command will request a password +1. Generate a password hash for PostgreSQL. This assumes you are using the default + username of `gitlab` (recommended). The command requests a password and confirmation. Use the value that is output by this command in the next step as the value of `POSTGRESQL_PASSWORD_HASH`. @@ -31,14 +31,12 @@ together with Omnibus GitLab. This is recommended as part of our - `POSTGRESQL_PASSWORD_HASH` - The value output from the previous step - `APPLICATION_SERVER_IP_BLOCKS` - A space delimited list of IP subnets or IP - addresses of the GitLab application servers that will connect to the + addresses of the GitLab application servers that connect to the database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` ```ruby # Disable all components except PostgreSQL - roles ['postgres_role'] - repmgr['enable'] = false - consul['enable'] = false + roles(['postgres_role']) prometheus['enable'] = false alertmanager['enable'] = false pgbouncer_exporter['enable'] = false @@ -59,12 +57,9 @@ together with Omnibus GitLab. This is recommended as part of our gitlab_rails['auto_migrate'] = false ``` - NOTE: - The role `postgres_role` was introduced with GitLab 10.3 - 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Note the PostgreSQL node's IP address or hostname, port, and - plain text password. These will be necessary when configuring the GitLab + plain text password. These are necessary when configuring the GitLab application servers later. 1. [Enable monitoring](replication_and_failover.md#enable-monitoring) diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index 4aa73212e43..533ebe0ad2f 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -27,7 +27,7 @@ be textually exported. This ensures that: To configure the Pseudonymizer, you need to: - Provide a manifest file that describes which fields should be included or - pseudonymized ([example `manifest.yml` file](https://gitlab.com/gitlab-org/gitlab/tree/master/config/pseudonymizer.yml)). + pseudonymized ([example `manifest.yml` file](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/pseudonymizer.yml)). A default manifest is provided with the GitLab installation, using a relative file path that resolves from the Rails root. Alternatively, you can use an absolute file path. - Use an object storage and specify the connection parameters in the `pseudonymizer.upload.connection` configuration option. diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index 8d2ca103c82..7f344a00f72 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -34,10 +34,11 @@ exactly which repositories are causing the trouble. - Receiving an error when trying to push code - `remote: error: cannot lock ref` - A 500 error when viewing the GitLab dashboard or when accessing a specific project. -### Check all GitLab repositories +### Check project code repositories -This task loops through all repositories on the GitLab server and runs the -integrity check described previously. +This task loops through the project code repositories and runs the integrity check +described previously. If a project uses a pool repository, that will also be checked. +Other types of Git repositories [are not checked](https://gitlab.com/gitlab-org/gitaly/-/issues/3643). **Omnibus Installation** @@ -246,6 +247,41 @@ end p "#{uploads_deleted} remote objects were destroyed." ``` +### Delete references to missing artifacts + +`gitlab-rake gitlab:artifacts:check VERBOSE=1` detects when artifacts (or `job.log` files): + +- Are deleted outside of GitLab. +- Have references still in the GitLab database. + +When this scenario is detected, the Rake task displays an error message. For example: + +```shell +Checking integrity of Job artifacts +- 3..8: Failures: 2 + - Job artifact: 3: # + - Job artifact: 8: # +Done! + +``` + +To delete these references to missing local artifacts (`job.log` files): + +1. Open the [GitLab Rails Console](../operations/rails_console.md#starting-a-rails-console-session). +1. Run the following Ruby code: + + ```ruby + artifacts_deleted = 0 + ::Ci::JobArtifact.all.each do |artifact| ### Iterate artifacts + # next if artifact.file.filename != "job.log" ### Uncomment if only `job.log` files' references are to be processed + next if artifact.file.exists? ### Skip if the file reference is valid + artifacts_deleted += 1 + puts "#{artifact.id} #{artifact.file.path} is missing." ### Allow verification before destroy + # artifact.destroy! ### Uncomment to actually destroy + end + puts "Count of identified/destroyed invalid references: #{artifacts_deleted}" + ``` + ### Delete references to missing LFS objects If `gitlab-rake gitlab:lfs:check VERBOSE=1` detects LFS objects that exist in the database diff --git a/doc/administration/read_only_gitlab.md b/doc/administration/read_only_gitlab.md index 698da80a07c..c8931eb79df 100644 --- a/doc/administration/read_only_gitlab.md +++ b/doc/administration/read_only_gitlab.md @@ -19,10 +19,10 @@ The configuration for doing so depends on your desired outcome. The first thing you'll want to accomplish is to ensure that no changes can be made to your repositories. There's two ways you can accomplish that: -- Either stop Unicorn/Puma to make the internal API unreachable: +- Either stop Puma to make the internal API unreachable: ```shell - sudo gitlab-ctl stop puma # or unicorn + sudo gitlab-ctl stop puma ``` - Or, open up a Rails console: @@ -46,19 +46,19 @@ made to your repositories. There's two ways you can accomplish that: ## Shut down the GitLab UI If you don't mind shutting down the GitLab UI, then the easiest approach is to -stop `sidekiq` and `puma`/`unicorn`, and you'll effectively ensure that no +stop `sidekiq` and `puma`, and you'll effectively ensure that no changes can be made to GitLab: ```shell sudo gitlab-ctl stop sidekiq -sudo gitlab-ctl stop puma # or unicorn +sudo gitlab-ctl stop puma ``` When you're ready to revert this: ```shell sudo gitlab-ctl start sidekiq -sudo gitlab-ctl start puma # or unicorn +sudo gitlab-ctl start puma ``` ## Make the database read-only diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index 20a9fbd7d68..9fde91903e8 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -42,7 +42,7 @@ There should be no more than one Sentinel on the same machine though. You also need to take into consideration the underlying network topology, making sure you have redundant connectivity between Redis / Sentinel and -GitLab instances, otherwise the networks will become a single point of +GitLab instances, otherwise the networks become a single point of failure. Running Redis in a scaled environment requires a few things: @@ -73,7 +73,7 @@ whole cluster down, invalidating the failover effort. ## Recommended setup -For a minimal setup, you will install the Omnibus GitLab package in `3` +For a minimal setup, you need to install the Omnibus GitLab package in `3` **independent** machines, both with **Redis** and **Sentinel**: - Redis Primary + Sentinel @@ -84,7 +84,7 @@ If you are not sure or don't understand why and where the amount of nodes come from, read [Redis setup overview](#redis-setup-overview) and [Sentinel setup overview](#sentinel-setup-overview). -For a recommended setup that can resist more failures, you will install +For a recommended setup that can resist more failures, you need to install the Omnibus GitLab package in `5` **independent** machines, both with **Redis** and **Sentinel**: @@ -99,9 +99,9 @@ the Omnibus GitLab package in `5` **independent** machines, both with You must have at least `3` Redis servers: `1` primary, `2` Replicas, and they need to each be on independent machines (see explanation above). -You can have additional Redis nodes, that will help survive a situation +You can have additional Redis nodes, that helps to survive a situation where more nodes goes down. Whenever there is only `2` nodes online, a failover -will not be initiated. +is not initiated. As an example, if you have `6` Redis nodes, a maximum of `3` can be simultaneously down. @@ -117,7 +117,7 @@ in a failover situation, any **Replica** can be promoted as the new **Primary** the Sentinel servers. The replication requires authentication, so you need to define a password to -protect all Redis nodes and the Sentinels. They will all share the same +protect all Redis nodes and the Sentinels. All of them share the same password, and all instances must be able to talk to each other over the network. @@ -130,7 +130,7 @@ of Sentinels agreeing a node is down) to be able to start a failover. Whenever the **quorum** is met, the **majority** of all known Sentinel nodes need to be available and reachable, so that they can elect the Sentinel **leader** -who will take all the decisions to restore the service availability by: +who takes all the decisions to restore the service availability by: - Promoting a new **Primary** - Reconfiguring the other **Replicas** and make them point to the new **Primary** @@ -150,7 +150,7 @@ consensus algorithm to be effective in the case of a failure. In a `3` nodes topology, you can only afford `1` Sentinel node going down. Whenever the **majority** of the Sentinels goes down, the network partition -protection prevents destructive actions and a failover **will not be started**. +protection prevents destructive actions and a failover **is not started**. Here are some examples: @@ -159,11 +159,11 @@ Here are some examples: The **Leader** election can sometimes fail the voting round when **consensus** is not achieved (see the odd number of nodes requirement above). In that case, -a new attempt will be made after the amount of time defined in +a new attempt is made after the amount of time defined in `sentinel['failover_timeout']` (in milliseconds). NOTE: -We will see where `sentinel['failover_timeout']` is defined later. +We can see where `sentinel['failover_timeout']` is defined later. The `failover_timeout` variable has a lot of different use cases. According to the official documentation: @@ -183,7 +183,7 @@ the official documentation: - The maximum time a failover in progress waits for all the replicas to be reconfigured as replicas of the new primary. However even after this time - the replicas will be reconfigured by the Sentinels anyway, but not with + the replicas are reconfigured by the Sentinels anyway, but not with the exact parallel-syncs progression as specified. ## Configuring Redis @@ -195,7 +195,7 @@ If you already have Redis installed and running, read how to [switch from a single-machine installation](#switching-from-an-existing-single-machine-installation). NOTE: -Redis nodes (both primary and replica) will need the same password defined in +Redis nodes (both primary and replica) need the same password defined in `redis['password']`. At any time during a failover the Sentinels can reconfigure a node and change its status from primary to replica and vice versa. @@ -218,14 +218,14 @@ The requirements for a Redis setup are the following: ### Switching from an existing single-machine installation -If you already have a single-machine GitLab install running, you will need to +If you already have a single-machine GitLab install running, you need to replicate from this machine first, before de-activating the Redis instance inside it. -Your single-machine install will be the initial **Primary**, and the `3` others +Your single-machine install is the initial **Primary**, and the `3` others should be configured as **Replica** pointing to this machine. -After replication catches up, you will need to stop services in the +After replication catches up, you need to stop services in the single-machine install, to rotate the **Primary** to one of the new nodes. Make the required changes in configuration and restart the new nodes again. @@ -259,7 +259,7 @@ If you fail to replicate first, you may loose data (unprocessed background jobs) # sure you add extra firewall rules to prevent unauthorized access. redis['bind'] = '10.0.0.1' - # Define a port so Redis can listen for TCP requests which will allow other + # Define a port so Redis can listen for TCP requests which allows other # machines to connect to it. redis['port'] = 6379 @@ -303,7 +303,7 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). # sure you add extra firewall rules to prevent unauthorized access. redis['bind'] = '10.0.0.2' - # Define a port so Redis can listen for TCP requests which will allow other + # Define a port so Redis can listen for TCP requests which allows other # machines to connect to it. redis['port'] = 6379 @@ -333,8 +333,8 @@ You can specify multiple roles like sentinel and Redis as: Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after -a failover, as the nodes will be managed by the Sentinels, and even after a -`gitlab-ctl reconfigure`, they will get their configuration restored by +a failover, as the nodes are managed by the Sentinels, and even after a +`gitlab-ctl reconfigure`, they get their configuration restored by the same Sentinels. ### Step 3. Configuring the Redis Sentinel instances @@ -342,7 +342,7 @@ the same Sentinels. NOTE: If you are using an external Redis Sentinel instance, be sure to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH +configuration. This parameter causes clients to report `NOAUTH Authentication required.`. [Redis Sentinel 3.2.x does not support password authentication](https://github.com/antirez/redis/issues/3279). @@ -362,8 +362,8 @@ multiple machines with the Sentinel daemon. --- -1. SSH into the server that will host Redis Sentinel. -1. **You can omit this step if the Sentinels will be hosted in the same node as +1. SSH into the server that hosts Redis Sentinel. +1. **You can omit this step if the Sentinels is hosted in the same node as the other Redis instances.** [Download/install](https://about.gitlab.com/install/) the @@ -389,7 +389,7 @@ multiple machines with the Sentinel daemon. # The IP of the primary Redis node. redis['master_ip'] = '10.0.0.1' - # Define a port so Redis can listen for TCP requests which will allow other + # Define a port so Redis can listen for TCP requests which allows other # machines to connect to it. redis['port'] = 6379 @@ -437,7 +437,7 @@ multiple machines with the Sentinel daemon. ## ## - The maximum time a failover in progress waits for all the replica to be ## reconfigured as replicas of the new primary. However even after this time - ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the replicas are reconfigured by the Sentinels anyway, but not with ## the exact parallel-syncs progression as specified. # sentinel['failover_timeout'] = 60000 ``` @@ -511,7 +511,7 @@ If you enable Monitoring, it must be enabled on **all** Redis servers. retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), } - # Set the network addresses that the exporters will listen on + # Set the network addresses that the exporters listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' ``` @@ -528,7 +528,7 @@ In a real world usage, you would also set up firewall rules to prevent unauthorized access from other machines and block traffic from the outside (Internet). -We will use the same `3` nodes with **Redis** + **Sentinel** topology +We use the same `3` nodes with **Redis** + **Sentinel** topology discussed in [Redis setup overview](#redis-setup-overview) and [Sentinel setup overview](#sentinel-setup-overview) documentation. @@ -540,11 +540,11 @@ Here is a list and description of each **machine** and the assigned **IP**: - `10.0.0.4`: GitLab application Please note that after the initial configuration, if a failover is initiated -by the Sentinel nodes, the Redis nodes will be reconfigured and the **Primary** -will change permanently (including in `redis.conf`) from one node to the other, +by the Sentinel nodes, the Redis nodes are reconfigured and the **Primary** +changes permanently (including in `redis.conf`) from one node to the other, until a new failover is initiated again. -The same thing will happen with `sentinel.conf` that will be overridden after the +The same thing happens with `sentinel.conf` that is overridden after the initial execution, after any new sentinel node starts watching the **Primary**, or a failover promotes a different **Primary** node. @@ -691,7 +691,7 @@ To make this work with Sentinel: ``` NOTE: -For each persistence class, GitLab will default to using the +For each persistence class, GitLab defaults to using the configuration specified in `gitlab_rails['redis_sentinels']` unless overridden by the previously described settings. @@ -726,7 +726,7 @@ redis_replica_role['enable'] = true # enable only one of them # When Redis primary or Replica role are enabled, the following services are # enabled/disabled. Note that if Redis and Sentinel roles are combined, both -# services will be enabled. +# services are enabled. # The following services are disabled sentinel['enable'] = false diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index 2716d9bba37..141da2f79ec 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -21,7 +21,7 @@ The following are the requirements for providing your own Redis instance: [requirements page](../../install/requirements.md). - Standalone Redis or Redis high availability with Sentinel are supported. Redis Cluster is not supported. -- Managed Redis from cloud providers such as AWS ElastiCache will work. If these +- Managed Redis from cloud providers such as AWS ElastiCache works fine. If these services support high availability, be sure it is **not** the Redis Cluster type. Note the Redis node's IP address or hostname, port, and password (if required). @@ -53,13 +53,13 @@ Note the Redis node's IP address or hostname, port, and password (if required). This is the documentation for configuring a scalable Redis setup when you have installed Redis all by yourself and not using the bundled one that comes with the Omnibus packages, although using the Omnibus GitLab packages is -highly recommend as we optimize them specifically for GitLab, and we will take +highly recommend as we optimize them specifically for GitLab, and we take care of upgrading Redis to the latest supported version. Note also that you may elect to override all references to `/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis settings outlined in -[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/blob/master/config/README.md). +[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/README.md). We cannot stress enough the importance of reading the [replication and failover](replication_and_failover.md) documentation of the @@ -76,7 +76,7 @@ requirements: (e.g., one from an internal network). - Since Redis 3.2, you must define a password to receive external connections (`requirepass`). -- If you are using Redis with Sentinel, you will also need to define the same +- If you are using Redis with Sentinel, you also need to define the same password for the replica password definition (`masterauth`) in the same instance. In addition, read the prerequisites as described in the @@ -176,7 +176,7 @@ primary with IP `10.0.0.1` (some settings might overlap with the primary): sentinel monitor gitlab-redis 10.0.0.1 6379 2 ## Define with `sentinel down-after-milliseconds` the time in `ms` - ## that an unresponsive server will be considered down. + ## that an unresponsive server is considered down. sentinel down-after-milliseconds gitlab-redis 10000 ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple @@ -197,7 +197,7 @@ primary with IP `10.0.0.1` (some settings might overlap with the primary): ## ## * The maximum time a failover in progress waits for all the replicas to be ## reconfigured as replicas of the new primary. However even after this time - ## the replicas will be reconfigured by the Sentinels anyway, but not with + ## the replicas are reconfigured by the Sentinels anyway, but not with ## the exact parallel-syncs progression as specified. sentinel failover_timeout 30000 ``` @@ -218,7 +218,7 @@ The following steps should be performed in the GitLab application server which ideally should not have Redis or Sentinels in the same machine: 1. Edit `/home/git/gitlab/config/resque.yml` following the example in - [resque.yml.example](https://gitlab.com/gitlab-org/gitlab/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to + [resque.yml.example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to the correct server credentials: ```yaml @@ -249,7 +249,7 @@ In a real world usage, you would also set up firewall rules to prevent unauthorized access from other machines, and block traffic from the outside ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)). -For this example, **Sentinel 1** will be configured in the same machine as the +For this example, **Sentinel 1** is configured in the same machine as the **Redis Primary**, **Sentinel 2** and **Sentinel 3** in the same machines as the **Replica 1** and **Replica 2** respectively. @@ -261,11 +261,11 @@ Here is a list and description of each **machine** and the assigned **IP**: - `10.0.0.4`: GitLab application Please note that after the initial configuration, if a failover is initiated -by the Sentinel nodes, the Redis nodes will be reconfigured and the **Primary** -will change permanently (including in `redis.conf`) from one node to the other, +by the Sentinel nodes, the Redis nodes are reconfigured and the **Primary** +changes permanently (including in `redis.conf`) from one node to the other, until a new failover is initiated again. -The same thing will happen with `sentinel.conf` that will be overridden after the +The same thing happens with `sentinel.conf` that is overridden after the initial execution, after any new sentinel node starts watching the **Primary**, or a failover promotes a different **Primary** node. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index e4a699b962c..4627b27a45e 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -17,29 +17,34 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|------------------|--------------|-----------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Consul* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis Sentinel - Cache** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | -| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| External load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Consul(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis - Cache(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis - Queues / Shared State(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis Sentinel - Cache(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| Redis Sentinel - Queues / Shared State(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | | Gitaly | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 10k @@ -157,7 +162,7 @@ To set up GitLab and its components to accommodate up to 10,000 users: provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -411,11 +416,6 @@ The following IPs will be used as an example: - `10.6.0.12`: Consul 2 - `10.6.0.13`: Consul 3 -NOTE: -The configuration processes for the other servers in your reference architecture will -use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect -with the other servers. - To configure Consul: 1. SSH in to the server that will host Consul. @@ -426,10 +426,9 @@ To configure Consul: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['consul_role'] + roles(['consul_role']) ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -446,7 +445,11 @@ To configure Consul: gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. @@ -538,6 +541,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 pgbouncer ``` +1. Generate a password hash for the PostgreSQL replication username/password pair. This assumes you will use the default + username of `gitlab_replicator` (recommended). The command will request a password + and a confirmation. Use the value that is output by this command in the next step + as the value of ``: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_replicator + ``` + 1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default username of `gitlab-consul` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -550,19 +562,21 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] + # Disable all components except Patroni and Consul + roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # Sets `max_replication_slots` to double the number of database nodes. + # Patroni uses one extra slot per node when initiating the replication. + patroni['postgresql']['max_replication_slots'] = 8 + + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 9 + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. patroni['postgresql']['max_connections'] = 500 @@ -570,7 +584,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true consul['services'] = %w(postgresql) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -580,6 +593,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' + # Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value + postgresql['sql_replication_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' @@ -603,9 +618,8 @@ PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` Like most failover handling methods, this has a small chance of leading to data loss. Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -620,21 +634,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH in to the **primary node**: - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Make sure the `pg_trgm` extension is enabled (it might already be): - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Exit the database prompt by typing `\q` and Enter. +SSH in to any of the Patroni nodes on the **primary site**: 1. Check the status of the leader and cluster: @@ -676,7 +676,7 @@ The following IPs will be used as an example: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -693,7 +693,6 @@ The following IPs will be used as an example: # Configure Consul agent consul['watchers'] = %w(postgresql) - consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -705,9 +704,8 @@ The following IPs will be used as an example: node_exporter['listen_address'] = '0.0.0.0:9100' ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -826,8 +824,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -849,7 +847,6 @@ a node and change its status from primary to replica (and vice versa). redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -861,19 +858,22 @@ a node and change its status from primary to replica (and vice versa). # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.51:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -886,8 +886,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -916,7 +916,6 @@ You can specify multiple roles, like sentinel and Redis, as: redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -928,21 +927,24 @@ You can specify multiple roles, like sentinel and Redis, as: # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.52:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -984,7 +986,7 @@ To configure the Sentinel Cache server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-cache' @@ -1048,7 +1050,6 @@ To configure the Sentinel Cache server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1065,9 +1066,8 @@ To configure the Sentinel Cache server: gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other Consul/Sentinel nodes, and @@ -1097,8 +1097,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1114,7 +1114,6 @@ a node and change its status from primary to replica (and vice versa). redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1131,14 +1130,13 @@ a node and change its status from primary to replica (and vice versa). gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes @@ -1151,8 +1149,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1175,7 +1173,6 @@ You can specify multiple roles, like sentinel and Redis, as: #redis['master_port'] = 6379 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1192,16 +1189,15 @@ You can specify multiple roles, like sentinel and Redis, as: gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -1243,7 +1239,7 @@ To configure the Sentinel Queues server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-persistent' @@ -1307,7 +1303,6 @@ To configure the Sentinel Queues server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1324,17 +1319,8 @@ To configure the Sentinel Queues server: gitlab_rails['auto_migrate'] = false ``` -1. To prevent database migrations from running on upgrade, run: - - ```shell - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` - - Only the primary GitLab application server should handle migrations. - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other Sentinel nodes, and @@ -1397,9 +1383,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except PostgreSQL and Consul - roles ['postgres_role'] - repmgr['enable'] = false - patroni['enable'] = false + roles(['postgres_role', 'consul_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -1409,7 +1393,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -1435,7 +1418,11 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Follow the [post configuration](#praefect-postgresql-post-configuration).
@@ -1533,19 +1520,18 @@ To configure the Praefect nodes, on each one: 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: ```ruby - # Avoid running unnecessary services on the Gitaly server + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1583,19 +1569,20 @@ To configure the Praefect nodes, on each one: # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { 'default' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' - }, + 'nodes' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '' + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '' + }, + } } } @@ -1612,11 +1599,25 @@ To configure the Praefect nodes, on each one: # END user configuration ``` - 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. Praefect requires to run some database migrations, much like the main GitLab application. For this + you should select **one Praefect node only to run the migrations**, AKA the _Deploy Node_. This node + must be configured first before the others as follows: + + 1. In the `/etc/gitlab/gitlab.rb` file, change the `praefect['auto_migrate']` setting value from `false` to `true` + + 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect and + to run the Praefect database migrations. - 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On all other Praefect nodes, [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configure Gitaly @@ -1660,21 +1661,17 @@ On each node: storage paths, enable the network listener, and to configure the token: ```ruby - # /etc/gitlab/gitlab.rb - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1682,9 +1679,11 @@ On each node: # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + # Gitaly + gitaly['enable'] = true + # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -1726,9 +1725,8 @@ On each node: }) ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1835,28 +1833,19 @@ To configure the Sidekiq nodes, on each one: 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false + # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - puma['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + nginx['enable'] = false - ######################################## - #### Redis ### - ######################################## - + # Redis ## Redis connection details ## First cluster that will host the cache gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' @@ -1888,13 +1877,10 @@ To configure the Sidekiq nodes, on each one: {host: '10.6.0.83', port: 26379}, ] - ####################################### - ### Gitaly ### - ####################################### - - # git_data_dirs get configured for the Praefect virtual storage - # Address is Internal Load Balancer for Praefect - # Token is praefect_external_token + # Gitaly Cluster + ## git_data_dirs get configured for the Praefect virtual storage + ## Address is Internal Load Balancer for Praefect + ## Token is praefect_external_token git_data_dirs({ "default" => { "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP @@ -1902,20 +1888,17 @@ To configure the Sidekiq nodes, on each one: } }) - ####################################### - ### Postgres ### - ####################################### + # PostgreSQL gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '' gitlab_rails['db_adapter'] = 'postgresql' gitlab_rails['db_encoding'] = 'unicode' - # Prevent database migrations from running on upgrade automatically + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### + # Sidekiq + sidekiqp['enable'] = true sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs @@ -1924,9 +1907,7 @@ To configure the Sidekiq nodes, on each one: # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - ####################################### - ### Monitoring configuration ### - ####################################### + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true @@ -1934,18 +1915,15 @@ To configure the Sidekiq nodes, on each one: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Set the network addresses that the exporters will listen on + ## Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - # Rails Status for prometheus + ## Add the monitoring node's IP address to the monitoring whitelist gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] - ############################# - ### Object storage ### - ############################# - - # This is an example for configuring Object Storage on GCP - # Replace this config with your chosen Object Storage provider as desired + # Object Storage + ## This is an example for configuring Object Storage on GCP + ## Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { 'provider' => 'Google', 'google_project' => '', @@ -1958,11 +1936,26 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -1993,9 +1986,6 @@ On each node perform the following: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` @@ -2017,7 +2007,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true sidekiq['enable'] = false @@ -2090,9 +2080,15 @@ On each node perform the following: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: @@ -2111,6 +2107,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. If you're [using NFS](#configure-nfs-optional): 1. If necessary, install the NFS client utility packages using the following commands: @@ -2150,7 +2160,8 @@ On each node perform the following: registry['gid'] = 9002 ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Confirm the node can connect to Gitaly: ```shell @@ -2214,55 +2225,34 @@ To configure the Monitoring node: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - external_url 'http://gitlab.example.com' + roles(['monitoring_role', 'consul_role']) - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true + # Grafana grafana['admin_password'] = '' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false + # Nginx - For Grafana access + nginx['enable'] = true ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. In the GitLab UI, set `admin/application_settings/metrics_and_profiling` > Metrics - Grafana to `/-/grafana` to `http[s]:///-/grafana` @@ -2395,35 +2385,46 @@ time use Google Cloud’s Kubernetes Engine (GKE) and associated machine types, and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. -| Service | Nodes | Configuration | GCP | Allocatable CPUs and Memory | -|-------------------------------------------------------|-------|-------------------------|------------------|-----------------------------| -| Webservice | 4 | 32 vCPU, 28.8 GB memory | `n1-standard-32` | 127.5 vCPU, 118 GB memory | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | 15.5 vCPU, 50 GB memory | -| Supporting services such as NGINX, Prometheus, etc... | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | 7.75 vCPU, 25 GB memory | +| Service | Nodes(1) | Configuration | GCP | Allocatable CPUs and Memory | +|-------------------------------------------------------|----------|-------------------------|------------------|-----------------------------| +| Webservice | 4 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | 127.5 vCPU, 118 GB memory | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | 15.5 vCPU, 50 GB memory | +| Supporting services such as NGINX, Prometheus, etc. | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | 7.75 vCPU, 25 GB memory | + + + +1. Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. + In production deployments there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): | Service | Nodes | Configuration | GCP | |--------------------------------------------|-------|-------------------------|------------------| -| Consul* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| PostgreSQL* | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | -| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | -| Redis Sentinel - Cache** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | -| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | +| Consul(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| PostgreSQL(1) | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| Internal load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| Redis - Cache(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis - Queues / Shared State(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis Sentinel - Cache(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | +| Redis Sentinel - Queues / Shared State(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | | Gitaly | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| Object storage | n/a | n/a | n/a | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| Object storage(4) | n/a | n/a | n/a | + + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 10k diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 129386d6ce5..1f72c45c2b7 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -17,29 +17,34 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |------------------------------------------|-------------|-------------------------|------------------|--------------|-----------| -| External load balancing node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Consul* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 3 | 16 vCPU, 60 GB memory | `n1-standard-1` | `m5.4xlarge` | `D16s v3` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.large` | `F2s v2` | -| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis Sentinel - Cache** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | -| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| External load balancing node(3) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Consul(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 3 | 16 vCPU, 60 GB memory | `n1-standard-1` | `m5.4xlarge` | `D16s v3` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node(3) | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.large` | `F2s v2` | +| Redis - Cache(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis - Queues / Shared State(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis Sentinel - Cache(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| Redis Sentinel - Queues / Shared State(2)| 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | | Gitaly | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | | Praefect | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 25k @@ -157,7 +162,7 @@ To set up GitLab and its components to accommodate up to 25,000 users: provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -413,11 +418,6 @@ The following IPs will be used as an example: - `10.6.0.12`: Consul 2 - `10.6.0.13`: Consul 3 -NOTE: -The configuration processes for the other servers in your reference architecture will -use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect -with the other servers. - To configure Consul: 1. SSH in to the server that will host Consul. @@ -428,10 +428,9 @@ To configure Consul: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['consul_role'] + roles(['consul_role']) ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -448,7 +447,11 @@ To configure Consul: gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. @@ -540,6 +543,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 pgbouncer ``` +1. Generate a password hash for the PostgreSQL replication username/password pair. This assumes you will use the default + username of `gitlab_replicator` (recommended). The command will request a password + and a confirmation. Use the value that is output by this command in the next step + as the value of ``: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_replicator + ``` + 1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default username of `gitlab-consul` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -552,19 +564,21 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] + # Disable all components except Patroni and Consul + roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # Sets `max_replication_slots` to double the number of database nodes. + # Patroni uses one extra slot per node when initiating the replication. + patroni['postgresql']['max_replication_slots'] = 8 + + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 9 + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. patroni['postgresql']['max_connections'] = 500 @@ -572,7 +586,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true consul['services'] = %w(postgresql) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -582,6 +595,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' + # Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value + postgresql['sql_replication_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' @@ -605,9 +620,8 @@ PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` Like most failover handling methods, this has a small chance of leading to data loss. Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -622,21 +636,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH in to the **primary node**: - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Make sure the `pg_trgm` extension is enabled (it might already be): - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Exit the database prompt by typing `\q` and Enter. +SSH in to any of the Patroni nodes on the **primary site**: 1. Check the status of the leader and cluster: @@ -678,7 +678,7 @@ The following IPs will be used as an example: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -695,7 +695,6 @@ The following IPs will be used as an example: # Configure Consul agent consul['watchers'] = %w(postgresql) - consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -707,9 +706,8 @@ The following IPs will be used as an example: node_exporter['listen_address'] = '0.0.0.0:9100' ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -828,8 +826,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role'] # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -851,7 +849,6 @@ a node and change its status from primary to replica (and vice versa). redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -863,19 +860,22 @@ a node and change its status from primary to replica (and vice versa). # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.51:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -888,8 +888,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role'] # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -918,7 +918,6 @@ You can specify multiple roles, like sentinel and Redis, as: redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -930,21 +929,25 @@ You can specify multiple roles, like sentinel and Redis, as: # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.52:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -986,7 +989,7 @@ To configure the Sentinel Cache server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-cache' @@ -1050,7 +1053,6 @@ To configure the Sentinel Cache server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1067,11 +1069,11 @@ To configure the Sentinel Cache server: gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. @@ -1099,8 +1101,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1116,7 +1118,6 @@ a node and change its status from primary to replica (and vice versa). redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1133,14 +1134,13 @@ a node and change its status from primary to replica (and vice versa). gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes @@ -1153,8 +1153,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1177,7 +1177,6 @@ You can specify multiple roles, like sentinel and Redis, as: #redis['master_port'] = 6379 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1189,21 +1188,25 @@ You can specify multiple roles, like sentinel and Redis, as: # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.62:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -1245,7 +1248,7 @@ To configure the Sentinel Queues server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-persistent' @@ -1309,7 +1312,6 @@ To configure the Sentinel Queues server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1326,7 +1328,10 @@ To configure the Sentinel Queues server: gitlab_rails['auto_migrate'] = false ``` -1. To prevent database migrations from running on upgrade, run: +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell sudo touch /etc/gitlab/skip-auto-reconfigure @@ -1334,11 +1339,8 @@ To configure the Sentinel Queues server: Only the primary GitLab application server should handle migrations. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. - 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Sentinel nodes, and make sure you set up the correct IPs. @@ -1399,9 +1401,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except PostgreSQL and Consul - roles ['postgres_role'] - repmgr['enable'] = false - patroni['enable'] = false + roles(['postgres_role', 'consul_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -1411,7 +1411,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -1437,7 +1436,11 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Follow the [post configuration](#praefect-postgresql-post-configuration).
@@ -1535,19 +1538,18 @@ To configure the Praefect nodes, on each one: 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: ```ruby - # Avoid running unnecessary services on the Gitaly server + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1585,19 +1587,20 @@ To configure the Praefect nodes, on each one: # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { 'default' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' - }, + 'nodes' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '' + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '' + }, + } } } @@ -1614,11 +1617,25 @@ To configure the Praefect nodes, on each one: # END user configuration ``` - 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace +the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. Praefect requires to run some database migrations, much like the main GitLab application. For this + you should select **one Praefect node only to run the migrations**, AKA the _Deploy Node_. This node + must be configured first before the others as follows: + + 1. In the `/etc/gitlab/gitlab.rb` file, change the `praefect['auto_migrate']` setting value from `false` to `true` + + 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: - 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect and + to run the Praefect database migrations. + +1. On all other Praefect nodes, [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configure Gitaly @@ -1662,21 +1679,17 @@ On each node: storage paths, enable the network listener, and to configure the token: ```ruby - # /etc/gitlab/gitlab.rb - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1684,9 +1697,11 @@ On each node: # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + # Gitaly + gitaly['enable'] = true + # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -1728,9 +1743,8 @@ On each node: }) ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1837,28 +1851,19 @@ To configure the Sidekiq nodes, on each one: 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false + # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - puma['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + nginx['enable'] = false - ######################################## - #### Redis ### - ######################################## - + # Redis ## Redis connection details ## First cluster that will host the cache gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' @@ -1890,13 +1895,10 @@ To configure the Sidekiq nodes, on each one: {host: '10.6.0.83', port: 26379}, ] - ####################################### - ### Gitaly ### - ####################################### - - # git_data_dirs get configured for the Praefect virtual storage - # Address is Internal Load Balancer for Praefect - # Token is praefect_external_token + # Gitaly Cluster + ## git_data_dirs get configured for the Praefect virtual storage + ## Address is Internal Load Balancer for Praefect + ## Token is praefect_external_token git_data_dirs({ "default" => { "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP @@ -1904,20 +1906,17 @@ To configure the Sidekiq nodes, on each one: } }) - ####################################### - ### Postgres ### - ####################################### + # PostgreSQL gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '' gitlab_rails['db_adapter'] = 'postgresql' gitlab_rails['db_encoding'] = 'unicode' - # Prevent database migrations from running on upgrade automatically + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### + # Sidekiq + sidekiq['enable'] = true sidekiq['listen_address'] = "0.0.0.0" # Set number of Sidekiq queue processes to the same number as available CPUs @@ -1926,9 +1925,7 @@ To configure the Sidekiq nodes, on each one: # Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - ####################################### - ### Monitoring configuration ### - ####################################### + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true @@ -1936,16 +1933,13 @@ To configure the Sidekiq nodes, on each one: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Set the network addresses that the exporters will listen on + ## Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - # Rails Status for prometheus + ## Add the monitoring node's IP address to the monitoring whitelist gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] - ############################# - ### Object storage ### - ############################# - + # Object Storage # This is an example for configuring Object Storage on GCP # Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { @@ -1960,11 +1954,26 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -1997,9 +2006,6 @@ On each node perform the following: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` @@ -2021,7 +2027,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true sidekiq['enable'] = false @@ -2094,9 +2100,15 @@ On each node perform the following: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: @@ -2115,6 +2127,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. If you're [using NFS](#configure-nfs-optional): 1. If necessary, install the NFS client utility packages using the following commands: @@ -2154,7 +2180,7 @@ On each node perform the following: registry['gid'] = 9002 ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Confirm the node can connect to Gitaly: ```shell @@ -2218,52 +2244,30 @@ To configure the Monitoring node: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - external_url 'http://gitlab.example.com' + roles(['monitoring_role', 'consul_role']) - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true + # Grafana grafana['admin_password'] = '' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false + # Nginx - For Grafana access + nginx['enable'] = true ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 69e261cfbe6..7db3a343e0b 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -18,20 +18,24 @@ For a full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |------------------------------------------|--------|-------------------------|-----------------|--------------|----------| -| Load balancer | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Redis** | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | +| Load balancer(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Redis(2) | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | | Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | | NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + +1. Can be optionally run on reputable third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run as reputable third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run as reputable third party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 2k @@ -84,7 +88,7 @@ To set up GitLab and its components to accommodate up to 2,000 users: 1. [Configure Gitaly](#configure-gitaly), which provides access to the Git repositories. 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -265,10 +269,8 @@ further configuration steps. database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` ```ruby - # Disable all components except PostgreSQL - roles ['postgres_role'] - patroni['enable'] = false - consul['enable'] = false + # Disable all components except PostgreSQL related ones + roles(['postgres_role']) prometheus['enable'] = false alertmanager['enable'] = false pgbouncer_exporter['enable'] = false @@ -295,6 +297,9 @@ further configuration steps. gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Note the PostgreSQL node's IP address or hostname, port, and plain text password. These will be necessary when configuring the [GitLab @@ -346,20 +351,18 @@ Omnibus: ```ruby ## Enable Redis redis['enable'] = true - - ## Disable all other services + + # Avoid running unnecessary services on the Redis server + gitaly['enable'] = false + postgresql['enable'] = false + puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - postgresql['enable'] = false - nginx['enable'] = false prometheus['enable'] = false alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - gitlab_exporter['enable'] = false - gitaly['enable'] = false grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false redis['bind'] = '0.0.0.0' redis['port'] = 6379 @@ -376,7 +379,11 @@ Omnibus: } ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Note the Redis node's IP address or hostname, port, and Redis password. These will be necessary when [configuring the GitLab application servers](#configure-gitlab-rails) later. @@ -455,16 +462,14 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -472,9 +477,11 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + # Gitaly + gitaly['enable'] = true + # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -494,7 +501,11 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: }) ``` -1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Confirm that Gitaly can perform callbacks to the internal API: ```shell @@ -629,7 +640,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true @@ -658,10 +669,7 @@ On each node perform the following: gitlab_rails['monitoring_whitelist'] = ['/32', '127.0.0.0/8'] nginx['status']['options']['allow'] = ['/32', '127.0.0.0/8'] - ############################# - ### Object storage ### - ############################# - + # Object Storage # This is an example for configuring Object Storage on GCP # Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { @@ -677,6 +685,13 @@ On each node perform the following: gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" + ## Uncomment and edit the following options if you have set up NFS ## ## Prevent GitLab from starting if NFS data mounts are not available @@ -710,7 +725,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. 1. Tail the logs to see the requests: @@ -718,11 +746,6 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` -1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two - application nodes and install it on the other application node and the - [Gitaly node](#configure-gitaly) and - [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see @@ -765,38 +788,21 @@ running [Prometheus](../monitoring/prometheus/index.md) and 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby + roles(['monitoring_role']) + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana + # Grafana grafana['enable'] = true - grafana['admin_password'] = 'toomanysecrets' + grafana['admin_password'] = '' + grafana['disable_login_form'] = false - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false + # Nginx - For Grafana access nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false - - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false ``` 1. Prometheus also needs some scrape configurations to pull all the data from the various diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index d81f98a35d4..bca5e4c3dab 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -27,26 +27,31 @@ For a full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-----------------------|-----------------|--------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis** | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul* + Sentinel** | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| External load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis(2) | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul(1) + Sentinel(2) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Gitaly | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | | GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | | NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 3k @@ -169,7 +174,7 @@ To set up GitLab and its components to accommodate up to 3,000 users: provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -470,8 +475,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -487,7 +492,6 @@ a node and change its status from primary to replica (and vice versa). redis['password'] = 'redis-password-goes-here' ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -508,6 +512,9 @@ a node and change its status from primary to replica (and vice versa). gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: @@ -546,8 +553,8 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -570,7 +577,6 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s #redis['master_port'] = 6379 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -591,12 +597,15 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -638,7 +647,7 @@ To configure the Sentinel: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role', 'consul_role'] + roles(['redis_sentinel_role', 'consul_role']) # Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis' @@ -702,7 +711,6 @@ To configure the Sentinel: # sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -720,7 +728,11 @@ To configure the Sentinel: gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. @@ -813,6 +825,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 pgbouncer ``` +1. Generate a password hash for the PostgreSQL replication username/password pair. This assumes you will use the default + username of `gitlab_replicator` (recommended). The command will request a password + and a confirmation. Use the value that is output by this command in the next step + as the value of ``: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_replicator + ``` + 1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default username of `gitlab-consul` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -825,27 +846,28 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] - + # Disable all components except Patroni and Consul + roles(['patroni_role']) + # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # Sets `max_replication_slots` to double the number of database nodes. + # Patroni uses one extra slot per node when initiating the replication. + patroni['postgresql']['max_replication_slots'] = 6 + + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 7 + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. patroni['postgresql']['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - + # Configure the Consul agent - consul['enable'] = true consul['services'] = %w(postgresql) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -855,6 +877,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' + # Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value + postgresql['sql_replication_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' @@ -878,9 +902,8 @@ PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` Like most failover handling methods, this has a small chance of leading to data loss. Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -895,22 +918,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH in to the **primary node**: - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Enable the `pg_trgm` and `btree_gist` extensions: - - ```shell - CREATE EXTENSION pg_trgm; - CREATE EXTENSION btree_gist; - ``` - -1. Exit the database prompt by typing `\q` and Enter. +SSH in to any of the Patroni nodes on the **primary site**: 1. Check the status of the leader and cluster: @@ -952,7 +960,7 @@ The following IPs will be used as an example: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -969,7 +977,6 @@ The following IPs will be used as an example: # Configure Consul agent consul['watchers'] = %w(postgresql) - consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -982,6 +989,9 @@ The following IPs will be used as an example: pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Create a `.pgpass` file so Consul is able to @@ -1097,9 +1107,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except PostgreSQL and Consul - roles ['postgres_role'] - repmgr['enable'] = false - patroni['enable'] = false + roles(['postgres_role', 'consul_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -1109,7 +1117,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -1135,6 +1142,9 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Follow the [post configuration](#praefect-postgresql-post-configuration). @@ -1213,7 +1223,7 @@ Praefect requires several secret tokens to secure communications across the Clus Gitaly Cluster nodes are configured in Praefect via a `virtual storage`. Each storage contains the details of each Gitaly node that makes up the cluster. Each storage is also given a name -and this name is used in several areas of the config. In this guide, the name of the storage will be +and this name is used in several areas of the configuration. In this guide, the name of the storage will be `default`. Also, this guide is geared towards new installs, if upgrading an existing environment to use Gitaly Cluster, you may need to use a different name. Refer to the [Praefect documentation](../gitaly/praefect.md#praefect) for more info. @@ -1233,19 +1243,18 @@ To configure the Praefect nodes, on each one: 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: ```ruby - # Avoid running unnecessary services on the Gitaly server + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1283,19 +1292,20 @@ To configure the Praefect nodes, on each one: # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { 'default' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' - }, + 'nodes' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '' + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '' + }, + } } } @@ -1312,11 +1322,25 @@ To configure the Praefect nodes, on each one: # END user configuration ``` - 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace +the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Praefect requires to run some database migrations, much like the main GitLab application. For this + you should select **one Praefect node only to run the migrations**, AKA the _Deploy Node_. This node + must be configured first before the others as follows: + + 1. In the `/etc/gitlab/gitlab.rb` file, change the `praefect['auto_migrate']` setting value from `false` to `true` + + 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect and + to run the Praefect database migrations. + +1. On all other Praefect nodes, [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configure Gitaly @@ -1360,29 +1384,27 @@ On each node: storage paths, enable the network listener, and to configure the token: ```ruby - # /etc/gitlab/gitlab.rb - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false + # Gitaly + gitaly['enable'] = true + # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' # Make Gitaly accept connections on all network interfaces. You must use @@ -1426,9 +1448,8 @@ On each node: }) ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1537,29 +1558,19 @@ To configure the Sidekiq nodes, one each one: 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false + # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - puma['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + nginx['enable'] = false - ######################################## - #### Redis ### - ######################################## - - ## Must be the same in every sentinel node + # Redis redis['master_name'] = 'gitlab-redis' ## The same password for Redis authentication you set up for the master node. @@ -1572,13 +1583,10 @@ To configure the Sidekiq nodes, one each one: {'host' => '10.6.0.13', 'port' => 26379}, ] - ####################################### - ### Gitaly ### - ####################################### - - # git_data_dirs get configured for the Praefect virtual storage - # Address is Internal Load Balancer for Praefect - # Token is praefect_external_token + # Gitaly Cluster + ## git_data_dirs get configured for the Praefect virtual storage + ## Address is Internal Load Balancer for Praefect + ## Token is praefect_external_token git_data_dirs({ "default" => { "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP @@ -1586,31 +1594,26 @@ To configure the Sidekiq nodes, one each one: } }) - ####################################### - ### Postgres ### - ####################################### + # PostgreSQL gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '' gitlab_rails['db_adapter'] = 'postgresql' gitlab_rails['db_encoding'] = 'unicode' - # Prevent database migrations from running on upgrade automatically + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### + # Sidekiq + sidekiq['enable'] = true sidekiq['listen_address'] = "0.0.0.0" - # Set number of Sidekiq queue processes to the same number as available CPUs + ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 2 - # Set number of Sidekiq threads per queue process to the recommend number of 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - ####################################### - ### Monitoring configuration ### - ####################################### + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true @@ -1618,19 +1621,16 @@ To configure the Sidekiq nodes, one each one: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Set the network addresses that the exporters will listen on + ## Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - # Rails Status for prometheus + ## Add the monitoring node's IP address to the monitoring whitelist gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] gitlab_rails['prometheus_address'] = '10.6.0.81:9090' - ############################# - ### Object storage ### - ############################# - - # This is an example for configuring Object Storage on GCP - # Replace this config with your chosen Object Storage provider as desired + # Object Storage + ## This is an example for configuring Object Storage on GCP + ## Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { 'provider' => 'Google', 'google_project' => '', @@ -1644,9 +1644,28 @@ To configure the Sidekiq nodes, one each one: gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Verify the GitLab services are running: ```shell @@ -1728,7 +1747,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true sidekiq['enable'] = false @@ -1793,10 +1812,7 @@ On each node perform the following: #registry['uid'] = 9002 #registry['gid'] = 9002 - ############################# - ### Object storage ### - ############################# - + # Object storage # This is an example for configuring Object Storage on GCP # Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { @@ -1811,6 +1827,13 @@ On each node perform the following: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` 1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the @@ -1831,7 +1854,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. 1. Tail the logs to see the requests: @@ -1839,11 +1875,6 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` -1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two - application nodes and install it on the other application node, the - [Gitaly node](#configure-gitaly) and the [Sidekiq node](#configure-sidekiq) and - [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - 1. Verify the GitLab services are running: ```shell @@ -1902,45 +1933,26 @@ running [Prometheus](../monitoring/prometheus/index.md) and 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - external_url 'http://gitlab.example.com' + roles(['monitoring_role', 'consul_role']) - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true + # Grafana grafana['admin_password'] = '' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false + # Nginx - For Grafana access + nginx['enable'] = true ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -2074,7 +2086,7 @@ but with smaller performance requirements, several modifications can be consider - PostgreSQL: Can be run on reputable Cloud PaaS solutions such as Google Cloud SQL or AWS RDS. In this setup, the PgBouncer and Consul nodes are no longer required: - Consul may still be desired if [Prometheus](../monitoring/prometheus/index.md) auto discovery is a requirement, otherwise you would need to [manually add scrape configurations](../monitoring/prometheus/index.md#adding-custom-scrape-configurations) for all nodes. - As Redis Sentinel runs on the same box as Consul in this architecture, it may need to be run on a separate box if Redis is still being run via Omnibus. - - Redis: Can be run on reputable Cloud PaaS solutions such as Google Memorystore and AWS Elasticache. In this setup, the Redis Sentinel is no longer required. + - Redis: Can be run on reputable Cloud PaaS solutions such as Google Memorystore and AWS ElastiCache. In this setup, the Redis Sentinel is no longer required.
diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 5cc463c953d..b3324cb75fb 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -17,29 +17,34 @@ full list of reference architectures, see | Service | Nodes | Configuration | GCP | AWS | Azure | |------------------------------------------|-------------|-------------------------|------------------|---------------|-----------| -| External load balancing node | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Consul* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis Sentinel - Cache** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | -| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| External load balancing node(3) | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Consul(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node(3) | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Redis - Cache(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis - Queues / Shared State(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis Sentinel - Cache(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | +| Redis Sentinel - Queues / Shared State(2)| 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `c5.large` | `A1 v2` | | Gitaly | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | `D64s v3` | | Praefect | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | | GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | -| NFS server | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | +| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 50k @@ -157,7 +162,7 @@ To set up GitLab and its components to accommodate up to 50,000 users: provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -420,11 +425,6 @@ The following IPs will be used as an example: - `10.6.0.12`: Consul 2 - `10.6.0.13`: Consul 3 -NOTE: -The configuration processes for the other servers in your reference architecture will -use the `/etc/gitlab/gitlab-secrets.json` file from your Consul server to connect -with the other servers. - To configure Consul: 1. SSH in to the server that will host Consul. @@ -435,10 +435,9 @@ To configure Consul: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['consul_role'] + roles(['consul_role']) ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -455,7 +454,11 @@ To configure Consul: gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. @@ -547,6 +550,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 pgbouncer ``` +1. Generate a password hash for the PostgreSQL replication username/password pair. This assumes you will use the default + username of `gitlab_replicator` (recommended). The command will request a password + and a confirmation. Use the value that is output by this command in the next step + as the value of ``: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_replicator + ``` + 1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default username of `gitlab-consul` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -559,19 +571,21 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] + # Disable all components except Patroni and Consul + roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # Sets `max_replication_slots` to double the number of database nodes. + # Patroni uses one extra slot per node when initiating the replication. + patroni['postgresql']['max_replication_slots'] = 8 + + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 9 + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. patroni['postgresql']['max_connections'] = 500 @@ -589,6 +603,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' + # Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value + postgresql['sql_replication_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' @@ -612,9 +628,8 @@ PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` Like most failover handling methods, this has a small chance of leading to data loss. Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -629,21 +644,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH in to the **primary node**: - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Make sure the `pg_trgm` extension is enabled (it might already be): - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Exit the database prompt by typing `\q` and Enter. +SSH in to any of the Patroni nodes on the **primary site**: 1. Check the status of the leader and cluster: @@ -685,7 +686,7 @@ The following IPs will be used as an example: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -702,7 +703,6 @@ The following IPs will be used as an example: # Configure Consul agent consul['watchers'] = %w(postgresql) - consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -714,9 +714,8 @@ The following IPs will be used as an example: node_exporter['listen_address'] = '0.0.0.0:9100' ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -835,8 +834,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -858,7 +857,6 @@ a node and change its status from primary to replica (and vice versa). redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -870,19 +868,22 @@ a node and change its status from primary to replica (and vice versa). # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' - + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.51:6379', + 'redis.password' => 'redis-password-goes-here', + } + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -895,8 +896,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -925,7 +926,6 @@ You can specify multiple roles, like sentinel and Redis, as: redis['maxmemory_samples'] = 5 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -937,21 +937,25 @@ You can specify multiple roles, like sentinel and Redis, as: # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' redis_exporter['listen_address'] = '0.0.0.0:9121' + redis_exporter['flags'] = { + 'redis.addr' => 'redis://10.6.0.52:6379', + 'redis.password' => 'redis-password-goes-here', + } # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -993,7 +997,7 @@ To configure the Sentinel Cache server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-cache' @@ -1057,7 +1061,6 @@ To configure the Sentinel Cache server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1074,11 +1077,11 @@ To configure the Sentinel Cache server: gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. @@ -1106,8 +1109,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1123,7 +1126,6 @@ a node and change its status from primary to replica (and vice versa). redis['password'] = 'REDIS_PRIMARY_PASSWORD_OF_SECOND_CLUSTER' ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1140,9 +1142,8 @@ a node and change its status from primary to replica (and vice versa). gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -1160,8 +1161,8 @@ You can specify multiple roles, like sentinel and Redis, as: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -1184,7 +1185,6 @@ You can specify multiple roles, like sentinel and Redis, as: #redis['master_port'] = 6379 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1201,16 +1201,16 @@ You can specify multiple roles, like sentinel and Redis, as: gitlab_rails['auto_migrate'] = false ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -1252,7 +1252,7 @@ To configure the Sentinel Queues server: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role'] + roles(['redis_sentinel_role', 'consul_role']) ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis-persistent' @@ -1316,7 +1316,6 @@ To configure the Sentinel Queues server: #sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -1333,7 +1332,7 @@ To configure the Sentinel Queues server: gitlab_rails['auto_migrate'] = false ``` -1. To prevent database migrations from running on upgrade, run: +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell sudo touch /etc/gitlab/skip-auto-reconfigure @@ -1341,11 +1340,11 @@ To configure the Sentinel Queues server: Only the primary GitLab application server should handle migrations. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Go through the steps again for all the other Sentinel nodes, and make sure you set up the correct IPs. @@ -1406,9 +1405,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except PostgreSQL and Consul - roles ['postgres_role'] - repmgr['enable'] = false - patroni['enable'] = false + roles(['postgres_role', 'consul_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -1418,7 +1415,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -1444,7 +1440,11 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Follow the [post configuration](#praefect-postgresql-post-configuration).
@@ -1542,19 +1542,18 @@ To configure the Praefect nodes, on each one: 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: ```ruby - # Avoid running unnecessary services on the Gitaly server + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1592,19 +1591,20 @@ To configure the Praefect nodes, on each one: # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { 'default' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' - }, + 'nodes' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '' + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '' + }, + } } } @@ -1621,11 +1621,25 @@ To configure the Praefect nodes, on each one: # END user configuration ``` - 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace +the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Praefect requires to run some database migrations, much like the main GitLab application. For this + you should select **one Praefect node only to run the migrations**, AKA the _Deploy Node_. This node + must be configured first before the others as follows: + + 1. In the `/etc/gitlab/gitlab.rb` file, change the `praefect['auto_migrate']` setting value from `false` to `true` + + 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect and + to run the Praefect database migrations. + +1. On all other Praefect nodes, [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configure Gitaly @@ -1669,21 +1683,17 @@ On each node: storage paths, enable the network listener, and to configure the token: ```ruby - # /etc/gitlab/gitlab.rb - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1691,9 +1701,11 @@ On each node: # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + # Gitaly + gitaly['enable'] = true + # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -1735,9 +1747,8 @@ On each node: }) ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1844,28 +1855,19 @@ To configure the Sidekiq nodes, on each one: 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false + # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - puma['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + nginx['enable'] = false - ######################################## - #### Redis ### - ######################################## - + # Redis ## Redis connection details ## First cluster that will host the cache gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' @@ -1897,10 +1899,7 @@ To configure the Sidekiq nodes, on each one: {host: '10.6.0.83', port: 26379}, ] - ####################################### - ### Gitaly ### - ####################################### - + # Gitaly # git_data_dirs get configured for the Praefect virtual storage # Address is Internal Load Balancer for Praefect # Token is praefect_external_token @@ -1911,31 +1910,26 @@ To configure the Sidekiq nodes, on each one: } }) - ####################################### - ### Postgres ### - ####################################### + # PostgreSQL gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '' gitlab_rails['db_adapter'] = 'postgresql' gitlab_rails['db_encoding'] = 'unicode' - # Prevent database migrations from running on upgrade automatically + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### + # Sidekiq + sidekiq['enable'] = true sidekiq['listen_address'] = "0.0.0.0" - # Set number of Sidekiq queue processes to the same number as available CPUs + ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - # Set number of Sidekiq threads per queue process to the recommend number of 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - ####################################### - ### Monitoring configuration ### - ####################################### + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true @@ -1946,15 +1940,12 @@ To configure the Sidekiq nodes, on each one: # Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - # Rails Status for prometheus + ## Add the monitoring node's IP address to the monitoring whitelist gitlab_rails['monitoring_whitelist'] = ['10.6.0.151/32', '127.0.0.0/8'] - ############################# - ### Object storage ### - ############################# - - # This is an example for configuring Object Storage on GCP - # Replace this config with your chosen Object Storage provider as desired + # Object storage + ## This is an example for configuring Object Storage on GCP + ## Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { 'provider' => 'Google', 'google_project' => '', @@ -1967,11 +1958,26 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -2011,9 +2017,6 @@ On each node perform the following: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` @@ -2035,7 +2038,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true sidekiq['enable'] = false @@ -2108,9 +2111,15 @@ On each node perform the following: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. If you're using [Gitaly with TLS support](#gitaly-cluster-tls-support), make sure the `git_data_dirs` entry is configured with `tls` instead of `tcp`: @@ -2129,6 +2138,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. If you're [using NFS](#configure-nfs-optional): 1. If necessary, install the NFS client utility packages using the following commands: @@ -2168,7 +2191,7 @@ On each node perform the following: registry['gid'] = 9002 ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Confirm the node can connect to Gitaly: ```shell @@ -2232,52 +2255,30 @@ To configure the Monitoring node: 1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package of your choice. Be sure to follow _only_ installation steps 1 and 2 on the page. -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - external_url 'http://gitlab.example.com' + roles(['monitoring_role', 'consul_role']) - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true + # Grafana grafana['admin_password'] = '' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false + # Nginx - For Grafana access + nginx['enable'] = true ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 792dd7020c7..9952df196c9 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -24,26 +24,31 @@ costly-to-operate environment by using the | Service | Nodes | Configuration | GCP | AWS | Azure | |--------------------------------------------|-------------|-------------------------|-----------------|--------------|----------| -| External load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis** | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul* + Sentinel** | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL* | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| External load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis(2) | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul(1) + Sentinel(2) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL(1) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node(3) | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Gitaly | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | | GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | `F16s v2`| | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage | n/a | n/a | n/a | n/a | n/a | +| Object storage(4) | n/a | n/a | n/a | n/a | n/a | | NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + + NOTE: -Components marked with * can be optionally run on reputable -third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. -Components marked with ** can be optionally run on reputable -third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. ```plantuml @startuml 5k @@ -161,7 +166,7 @@ To set up GitLab and its components to accommodate up to 5,000 users: provides access to the Git repositories. 1. [Configure Sidekiq](#configure-sidekiq). 1. [Configure the main GitLab Rails application](#configure-gitlab-rails) - to run Puma/Unicorn, Workhorse, GitLab Shell, and to serve all frontend + to run Puma, Workhorse, GitLab Shell, and to serve all frontend requests (which include UI, API, and Git over HTTP/SSH). 1. [Configure Prometheus](#configure-prometheus) to monitor your GitLab environment. @@ -462,8 +467,8 @@ a node and change its status from primary to replica (and vice versa). 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + # Specify server role as 'redis_master_role' and enable Consul agent + roles(['redis_master_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -479,7 +484,6 @@ a node and change its status from primary to replica (and vice versa). redis['password'] = 'redis-password-goes-here' ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -500,10 +504,13 @@ a node and change its status from primary to replica (and vice versa). gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). You can list the current Redis Primary, Replica status via: @@ -538,8 +545,8 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - # Specify server role as 'redis_replica_role' - roles ['redis_replica_role'] + # Specify server role as 'redis_replica_role' and enable Consul agent + roles(['redis_replica_role', 'consul_role']) # IP address pointing to a local IP that the other machines can reach to. # You can also set bind to '0.0.0.0' which listen in all interfaces. @@ -562,7 +569,6 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s #redis['master_port'] = 6379 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -583,12 +589,15 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. You can specify multiple roles, like sentinel and Redis, as: -`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +`roles(['redis_sentinel_role', 'redis_master_role'])`. Read more about [roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after @@ -630,7 +639,7 @@ To configure the Sentinel: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - roles ['redis_sentinel_role', 'consul_role'] + roles(['redis_sentinel_role', 'consul_role']) # Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis' @@ -694,7 +703,6 @@ To configure the Sentinel: # sentinel['failover_timeout'] = 60000 ## Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true ## The IPs of the Consul server nodes @@ -712,6 +720,9 @@ To configure the Sentinel: gitlab_rails['auto_migrate'] = false ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. @@ -805,6 +816,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. sudo gitlab-ctl pg-password-md5 pgbouncer ``` +1. Generate a password hash for the PostgreSQL replication username/password pair. This assumes you will use the default + username of `gitlab_replicator` (recommended). The command will request a password + and a confirmation. Use the value that is output by this command in the next step + as the value of ``: + + ```shell + sudo gitlab-ctl pg-password-md5 gitlab_replicator + ``` + 1. Generate a password hash for the Consul database username/password pair. This assumes you will use the default username of `gitlab-consul` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next @@ -817,19 +837,21 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except PostgreSQL, Patroni, and Consul - roles ['postgres_role'] + # Disable all components except Patroni and Consul + roles(['patroni_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - # Enable Patroni - patroni['enable'] = true - # Set `max_wal_senders` to one more than the number of database nodes in the cluster. + # Sets `max_replication_slots` to double the number of database nodes. + # Patroni uses one extra slot per node when initiating the replication. + patroni['postgresql']['max_replication_slots'] = 8 + + # Set `max_wal_senders` to one more than the number of replication slots in the cluster. # This is used to prevent replication from using up all of the # available database connections. - patroni['postgresql']['max_wal_senders'] = 4 - patroni['postgresql']['max_replication_slots'] = 4 + patroni['postgresql']['max_wal_senders'] = 9 + # Incoming recommended value for max connections is 500. See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5691. patroni['postgresql']['max_connections'] = 500 @@ -837,7 +859,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true consul['services'] = %w(postgresql) ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -847,6 +868,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. # # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value postgresql['pgbouncer_user_password'] = '' + # Replace POSTGRESQL_REPLICATION_PASSWORD_HASH with a generated md5 value + postgresql['sql_replication_password'] = '' # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value postgresql['sql_user_password'] = '' @@ -870,9 +893,8 @@ PostgreSQL, with Patroni managing its failover, will default to use `pg_rewind` Like most failover handling methods, this has a small chance of leading to data loss. Learn more about the various [Patroni replication methods](../postgresql/replication_and_failover.md#selecting-the-appropriate-patroni-replication-method). -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and replace - the file of the same name on this server. If that file is not on this server, - add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -887,21 +909,7 @@ are supported and can be added if needed. #### PostgreSQL post-configuration -SSH in to the **primary node**: - -1. Open a database prompt: - - ```shell - gitlab-psql -d gitlabhq_production - ``` - -1. Enable the `pg_trgm` extension: - - ```shell - CREATE EXTENSION pg_trgm; - ``` - -1. Exit the database prompt by typing `\q` and Enter. +SSH in to any of the Patroni nodes on the **primary site**: 1. Check the status of the leader and cluster: @@ -943,7 +951,7 @@ The following IPs will be used as an example: ```ruby # Disable all components except Pgbouncer and Consul agent - roles ['pgbouncer_role'] + roles(['pgbouncer_role']) # Configure PgBouncer pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul) @@ -960,7 +968,6 @@ The following IPs will be used as an example: # Configure Consul agent consul['watchers'] = %w(postgresql) - consul['enable'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } @@ -973,6 +980,9 @@ The following IPs will be used as an example: pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Create a `.pgpass` file so Consul is able to @@ -1088,9 +1098,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. ```ruby # Disable all components except PostgreSQL and Consul - roles ['postgres_role'] - repmgr['enable'] = false - patroni['enable'] = false + roles(['postgres_role', 'consul_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -1100,7 +1108,6 @@ in the second step, do not supply the `EXTERNAL_URL` value. gitlab_rails['auto_migrate'] = false # Configure the Consul agent - consul['enable'] = true ## Enable service discovery for Prometheus consul['monitoring_service_discovery'] = true @@ -1126,7 +1133,11 @@ in the second step, do not supply the `EXTERNAL_URL` value. # END user configuration ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Follow the [post configuration](#praefect-postgresql-post-configuration).
@@ -1224,19 +1235,18 @@ To configure the Praefect nodes, on each one: 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: ```ruby - # Avoid running unnecessary services on the Gitaly server + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1274,19 +1284,20 @@ To configure the Praefect nodes, on each one: # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') praefect['virtual_storages'] = { 'default' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' - }, + 'nodes' => { + 'gitaly-1' => { + 'address' => 'tcp://10.6.0.91:8075', + 'token' => '' + }, + 'gitaly-2' => { + 'address' => 'tcp://10.6.0.92:8075', + 'token' => '' + }, + 'gitaly-3' => { + 'address' => 'tcp://10.6.0.93:8075', + 'token' => '' + }, + } } } @@ -1303,11 +1314,25 @@ To configure the Praefect nodes, on each one: # END user configuration ``` - 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace +the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. Praefect requires to run some database migrations, much like the main GitLab application. For this + you should select **one Praefect node only to run the migrations**, AKA the _Deploy Node_. This node + must be configured first before the others as follows: + + 1. In the `/etc/gitlab/gitlab.rb` file, change the `praefect['auto_migrate']` setting value from `false` to `true` + + 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect and + to run the Praefect database migrations. - 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On all other Praefect nodes, [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Configure Gitaly @@ -1351,21 +1376,17 @@ On each node: storage paths, enable the network listener, and to configure the token: ```ruby - # /etc/gitlab/gitlab.rb - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false - nginx['enable'] = false puma['enable'] = false - unicorn['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - grafana['enable'] = false - - # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1373,9 +1394,11 @@ On each node: # Configure the gitlab-shell API callback URL. Without this, `git push` will # fail. This can be your 'front door' GitLab URL or an internal load # balancer. - # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + # Gitaly + gitaly['enable'] = true + # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections @@ -1417,9 +1440,8 @@ On each node: }) ``` -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from your Consul server, and - then replace the file of the same name on this server. If that file isn't on - this server, add the file from your Consul server to this server. +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1526,28 +1548,19 @@ To configure the Sidekiq nodes, one each one: 1. Open `/etc/gitlab/gitlab.rb` with your editor: ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false + # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - puma['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + nginx['enable'] = false - ######################################## - #### Redis ### - ######################################## - + # Redis ## Must be the same in every sentinel node redis['master_name'] = 'gitlab-redis' @@ -1561,13 +1574,10 @@ To configure the Sidekiq nodes, one each one: {'host' => '10.6.0.13', 'port' => 26379}, ] - ####################################### - ### Gitaly ### - ####################################### - - # git_data_dirs get configured for the Praefect virtual storage - # Address is Internal Load Balancer for Praefect - # Token is praefect_external_token + # Gitaly Cluster + ## git_data_dirs get configured for the Praefect virtual storage + ## Address is Internal Load Balancer for Praefect + ## Token is praefect_external_token git_data_dirs({ "default" => { "gitaly_address" => "tcp://10.6.0.40:2305", # internal load balancer IP @@ -1575,31 +1585,26 @@ To configure the Sidekiq nodes, one each one: } }) - ####################################### - ### Postgres ### - ####################################### + # PostgreSQL gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '' gitlab_rails['db_adapter'] = 'postgresql' gitlab_rails['db_encoding'] = 'unicode' - # Prevent database migrations from running on upgrade automatically + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### + # Sidekiq + sidekiq['enable'] = true sidekiq['listen_address'] = "0.0.0.0" - # Set number of Sidekiq queue processes to the same number as available CPUs + ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - # Set number of Sidekiq threads per queue process to the recommend number of 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 10 sidekiq['max_concurrency'] = 10 - ####################################### - ### Monitoring configuration ### - ####################################### + # Monitoring consul['enable'] = true consul['monitoring_service_discovery'] = true @@ -1607,19 +1612,16 @@ To configure the Sidekiq nodes, one each one: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Set the network addresses that the exporters will listen on + ## Set the network addresses that the exporters will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - # Rails Status for prometheus + ## Add the monitoring node's IP address to the monitoring whitelist gitlab_rails['monitoring_whitelist'] = ['10.6.0.81/32', '127.0.0.0/8'] gitlab_rails['prometheus_address'] = '10.6.0.81:9090' - ############################# - ### Object storage ### - ############################# - - # This is an example for configuring Object Storage on GCP - # Replace this config with your chosen Object Storage provider as desired + # Object Storage + ## This is an example for configuring Object Storage on GCP + ## Replace this config with your chosen Object Storage provider as desired gitlab_rails['object_store']['connection'] = { 'provider' => 'Google', 'google_project' => '', @@ -1632,9 +1634,29 @@ To configure the Sidekiq nodes, one each one: gitlab_rails['object_store']['objects']['packages']['bucket'] = "" gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Verify the GitLab services are running: ```shell @@ -1716,7 +1738,7 @@ On each node perform the following: }) ## Disable components that will not be on the GitLab application server - roles ['application_role'] + roles(['application_role']) gitaly['enable'] = false nginx['enable'] = true sidekiq['enable'] = false @@ -1785,6 +1807,13 @@ On each node perform the following: gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "" gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "" + gitlab_rails['backup_upload_connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' + } + gitlab_rails['backup_upload_remote_directory'] = "" + ## Uncomment and edit the following options if you have set up NFS ## ## Prevent GitLab from starting if NFS data mounts are not available @@ -1819,7 +1848,20 @@ On each node perform the following: sudo cp cert.pem /etc/gitlab/trusted-certs/ ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace + the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. + +1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: + + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` + + Only a single designated node should handle migrations as detailed in the + [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. 1. Tail the logs to see the requests: @@ -1827,11 +1869,6 @@ On each node perform the following: sudo gitlab-ctl tail gitaly ``` -1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two - application nodes and install it on the other application node, the - [Gitaly node](#configure-gitaly) and the [Sidekiq node](#configure-sidekiq) and - [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - 1. Verify the GitLab services are running: ```shell @@ -1890,45 +1927,26 @@ running [Prometheus](../monitoring/prometheus/index.md) and 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: ```ruby - external_url 'http://gitlab.example.com' + roles(['monitoring_role', 'consul_role']) - # Disable all other services - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_exporter['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - puma['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - gitlab_exporter['enable'] = false + external_url 'http://gitlab.example.com' - # Enable Prometheus - prometheus['enable'] = true + # Prometheus prometheus['listen_address'] = '0.0.0.0:9090' prometheus['monitor_kubernetes'] = false - # Enable Login form - grafana['disable_login_form'] = false - - # Enable Grafana - grafana['enable'] = true + # Grafana grafana['admin_password'] = '' + grafana['disable_login_form'] = false # Enable service discovery for Prometheus - consul['enable'] = true consul['monitoring_service_discovery'] = true consul['configuration'] = { retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13) } - # Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false + # Nginx - For Grafana access + nginx['enable'] = true ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 6698737af6a..49024365e30 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -138,12 +138,12 @@ As long as at least one of each component is online and capable of handling the ### Automated database failover **(PREMIUM SELF)** > - Level of complexity: **High** -> - Required domain knowledge: PgBouncer, Repmgr or Patroni, shared storage, distributed systems +> - Required domain knowledge: PgBouncer, Patroni, shared storage, distributed systems By adding automatic failover for database systems, you can enable higher uptime with additional database nodes. This extends the default database with cluster management and failover policies. -[PgBouncer in conjunction with Repmgr or Patroni](../postgresql/replication_and_failover.md) +[PgBouncer in conjunction with Patroni](../postgresql/replication_and_failover.md) is recommended. ### Instance level replication with GitLab Geo **(PREMIUM SELF)** diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index d5f57965a80..4b07cff7de2 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -206,211 +206,8 @@ To make sure your configuration is correct: ## Troubleshooting Gitaly -If you have any problems when using standalone Gitaly nodes, first -[check all the versions are up to date](../gitaly/index.md#check-versions-when-using-standalone-gitaly-servers). - -### `gitaly-debug` - -The `gitaly-debug` command provides "production debugging" tools for Gitaly and Git -performance. It is intended to help production engineers and support -engineers investigate Gitaly performance problems. - -If you're using GitLab 11.6 or newer, this tool should be installed on -your GitLab / Gitaly server already at `/opt/gitlab/embedded/bin/gitaly-debug`. -If you're investigating an older GitLab version you can compile this -tool offline and copy the executable to your server: - -```shell -git clone https://gitlab.com/gitlab-org/gitaly.git -cd cmd/gitaly-debug -GOOS=linux GOARCH=amd64 go build -o gitaly-debug -``` - -To see the help page of `gitaly-debug` for a list of supported sub-commands, run: - -```shell -gitaly-debug -h -``` - -### Commits, pushes, and clones return a 401 - -```plaintext -remote: GitLab: 401 Unauthorized -``` - -You will need to sync your `gitlab-secrets.json` file with your GitLab -app nodes. - -### Client side gRPC logs - -Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC -client has its own log file which may contain useful information when -you are seeing Gitaly errors. You can control the log level of the -gRPC client with the `GRPC_LOG_LEVEL` environment variable. The -default level is `WARN`. - -You can run a gRPC trace with: - -```shell -sudo GRPC_TRACE=all GRPC_VERBOSITY=DEBUG gitlab-rake gitlab:gitaly:check -``` - -### Observing `gitaly-ruby` traffic - -[`gitaly-ruby`](../gitaly/configure_gitaly.md#gitaly-ruby) is an internal implementation detail of Gitaly, -so, there's not that much visibility into what goes on inside -`gitaly-ruby` processes. - -If you have Prometheus set up to scrape your Gitaly process, you can see -request rates and error codes for individual RPCs in `gitaly-ruby` by -querying `grpc_client_handled_total`. Strictly speaking, this metric does -not differentiate between `gitaly-ruby` and other RPCs, but in practice -(as of GitLab 11.9), all gRPC calls made by Gitaly itself are internal -calls from the main Gitaly process to one of its `gitaly-ruby` sidecars. - -Assuming your `grpc_client_handled_total` counter only observes Gitaly, -the following query shows you RPCs are (most likely) internally -implemented as calls to `gitaly-ruby`: - -```prometheus -sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 -``` - -### Repository changes fail with a `401 Unauthorized` error - -If you're running Gitaly on its own server and notice that users can -successfully clone and fetch repositories (via both SSH and HTTPS), but can't -push to them or make changes to the repository in the web UI without getting a -`401 Unauthorized` message, then it's possible Gitaly is failing to authenticate -with the other nodes due to having the wrong secrets file. - -Confirm the following are all true: - -- When any user performs a `git push` to any repository on this Gitaly node, it - fails with the following error (note the `401 Unauthorized`): - - ```shell - remote: GitLab: 401 Unauthorized - To - ! [remote rejected] branch-name -> branch-name (pre-receive hook declined) - error: failed to push some refs to '' - ``` - -- When any user adds or modifies a file from the repository using the GitLab - UI, it immediately fails with a red `401 Unauthorized` banner. -- Creating a new project and [initializing it with a README](../../user/project/working_with_projects.md#blank-projects) - successfully creates the project but doesn't create the README. -- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on an app node and reproducing the error, you get `401` errors - when reaching the [`/api/v4/internal/allowed`](../../development/internal_api.md) endpoint: - - ```shell - # api_json.log - { - "time": "2019-07-18T00:30:14.967Z", - "severity": "INFO", - "duration": 0.57, - "db": 0, - "view": 0.57, - "status": 401, - "method": "POST", - "path": "\/api\/v4\/internal\/allowed", - "params": [ - { - "key": "action", - "value": "git-receive-pack" - }, - { - "key": "changes", - "value": "REDACTED" - }, - { - "key": "gl_repository", - "value": "REDACTED" - }, - { - "key": "project", - "value": "\/path\/to\/project.git" - }, - { - "key": "protocol", - "value": "web" - }, - { - "key": "env", - "value": "{\"GIT_ALTERNATE_OBJECT_DIRECTORIES\":[],\"GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE\":[],\"GIT_OBJECT_DIRECTORY\":null,\"GIT_OBJECT_DIRECTORY_RELATIVE\":null}" - }, - { - "key": "user_id", - "value": "2" - }, - { - "key": "secret_token", - "value": "[FILTERED]" - } - ], - "host": "gitlab.example.com", - "ip": "REDACTED", - "ua": "Ruby", - "route": "\/api\/:version\/internal\/allowed", - "queue_duration": 4.24, - "gitaly_calls": 0, - "gitaly_duration": 0, - "correlation_id": "XPUZqTukaP3" - } - - # nginx_access.log - [IP] - - [18/Jul/2019:00:30:14 +0000] "POST /api/v4/internal/allowed HTTP/1.1" 401 30 "" "Ruby" - ``` - -To fix this problem, confirm that your `gitlab-secrets.json` file -on the Gitaly node matches the one on all other nodes. If it doesn't match, -update the secrets file on the Gitaly node to match the others, then -[reconfigure the node](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -### Command line tools cannot connect to Gitaly - -If you are having trouble connecting to a Gitaly node with command line (CLI) tools, and certain actions result in a `14: Connect Failed` error message, it means that gRPC cannot reach your Gitaly node. - -Verify that you can reach Gitaly via TCP: - -```shell -sudo gitlab-rake gitlab:tcp_check[GITALY_SERVER_IP,GITALY_LISTEN_PORT] -``` - -If the TCP connection fails, check your network settings and your firewall rules. If the TCP connection succeeds, your networking and firewall rules are correct. - -If you use proxy servers in your command line environment, such as Bash, these can interfere with your gRPC traffic. - -If you use Bash or a compatible command line environment, run the following commands to determine whether you have proxy servers configured: - -```shell -echo $http_proxy -echo $https_proxy -``` - -If either of these variables have a value, your Gitaly CLI connections may be getting routed through a proxy which cannot connect to Gitaly. - -To remove the proxy setting, run the following commands (depending on which variables had values): - -```shell -unset http_proxy -unset https_proxy -``` - -### Gitaly not listening on new address after reconfiguring - -When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. - -When this occurs, performing a `sudo gitlab-ctl restart` will resolve the issue. This will no longer be necessary after [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved. - -### Permission denied errors appearing in Gitaly logs when accessing repositories from a standalone Gitaly node - -If this error occurs even though file permissions are correct, it's likely that -the Gitaly node is experiencing -[clock drift](https://en.wikipedia.org/wiki/Clock_drift). - -Please ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time -server to keep them synchronized if possible. +For troubleshooting information, see Gitaly and Gitaly Cluster +[troubleshooting information](../gitaly/index.md). ## Troubleshooting the GitLab Rails application diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index f0b9daead69..aae5475312f 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Set up Postfix for incoming email +# Set up Postfix for incoming email **(FREE SELF)** This document will take you through the steps of setting up a basic Postfix mail server with IMAP authentication on Ubuntu, to be used with [incoming email](incoming_email.md). diff --git a/doc/administration/repository_checks.md b/doc/administration/repository_checks.md index d603e5d8c92..869b1e7068f 100644 --- a/doc/administration/repository_checks.md +++ b/doc/administration/repository_checks.md @@ -41,9 +41,12 @@ in the [`repocheck.log` file](logs.md#repochecklog) on disk: - `/var/log/gitlab/gitlab-rails` for Omnibus GitLab installations - `/home/git/gitlab/log` for installations from source -If the periodic repository check causes false alarms, you can clear all repository check states by -going to **Admin Area > Settings > Repository** -(`/admin/application_settings/repository`) and clicking **Clear all repository checks**. +If the periodic repository check causes false alarms, you can clear all repository check states by: + +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Repository** (`/admin/application_settings/repository`). +1. Expand the **Repository maintenance** section. +1. Select **Clear all repository checks**. ## Run a check manually diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index cea2144122b..a1391f3e0ed 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -7,13 +7,11 @@ type: reference, howto # Repository storage **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/4578) in GitLab 8.10. - GitLab stores [repositories](../user/project/repository/index.md) on repository storage. Repository storage is either: - A `gitaly_address`, which points to a [Gitaly node](gitaly/index.md). -- A `path`, which points directly a directory where the repositories are stored. This method is +- A `path`, which points directly to the directory where the repositories are stored. This method is deprecated and [scheduled to be removed](https://gitlab.com/gitlab-org/gitaly/-/issues/1690) in GitLab 14.0. @@ -142,8 +140,9 @@ Omnibus stores the repositories in a `repositories` subdirectory of the `git-dat After you [configure](#configure-repository-storage-paths) multiple repository storage paths, you can choose where new repositories are stored: -1. Go to the Admin Area (**{admin}**). -1. Go to **Settings > Repository** and expand the **Repository storage** section. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Repository** and expand the **Repository storage** + section. 1. Enter values in the **Storage nodes for new repositories** fields. 1. Select **Save changes**. @@ -156,5 +155,5 @@ often it is chosen. That is, `(storage weight) / (sum of all weights) * 100 = ch ## Move repositories -To move a repository to a different repository path, use the same process as -[migrating to Gitaly Cluster](gitaly/praefect.md#migrate-to-gitaly-cluster). +To move a repository to a different repository storage (for example, from `default` to `storage2`), use the +same process as [migrating to Gitaly Cluster](gitaly/praefect.md#migrate-to-gitaly-cluster). diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 21bb11226ce..f55bff1bf34 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -80,8 +80,8 @@ Administrators can look up a project's hashed path from its name or ID using: To look up a project's hash path in the Admin Area: -1. Go to the **Admin Area** (**{admin}**). -1. Go to **Overview > Projects** and select the project. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Overview > Projects** and select the project. The **Gitaly relative path** is displayed there and looks similar to: diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md index a97eff23c2f..95fef1255af 100644 --- a/doc/administration/sidekiq.md +++ b/doc/administration/sidekiq.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Configuring Sidekiq +# Configuring Sidekiq **(FREE SELF)** This section discusses how to configure an external Sidekiq instance using the bundled Sidekiq in the GitLab package. @@ -26,7 +26,7 @@ you want using steps 1 and 2 from the GitLab downloads page. ## Optional: Enable extra Sidekiq processes sidekiq_cluster['enable'] = true sidekiq['queue_groups'] = [ - "elastic_indexer", + "elastic_commit_indexer", "*" ] ``` @@ -187,4 +187,5 @@ gitlab_rails['monitoring_whitelist'] = ['10.10.1.42', '127.0.0.1'] Related Sidekiq configuration: 1. [Extra Sidekiq processes](operations/extra_sidekiq_processes.md) +1. [Extra Sidekiq routing](operations/extra_sidekiq_routing.md) 1. [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index 7492bf4457a..ebc1723076b 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Signing outgoing email with S/MIME +# Signing outgoing email with S/MIME **(FREE SELF)** Notification emails sent by GitLab can be signed with S/MIME for improved security. diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md index fcd2bbc035f..48b98156b4f 100644 --- a/doc/administration/static_objects_external_storage.md +++ b/doc/administration/static_objects_external_storage.md @@ -16,7 +16,8 @@ from an external storage, such as a Content Delivery Network (CDN). To configure external storage for static objects: -1. Go to **Admin Area > Settings > Repository**. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. In the left sidebar, select **Settings > Repository**. 1. Expand the **Repository static objects** section. 1. Enter the base URL and an arbitrary token. When you [set up external storage](#set-up-external-storage), use a script that sets these values as `ORIGIN_HOSTNAME` and `STORAGE_TOKEN`. diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 898d98495d9..388ae74f207 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -23,7 +23,7 @@ Use [external object storage](https://docs.gitlab.com/charts/advanced/external-o ## Disabling Terraform state To disable terraform state site-wide, follow the steps below. -A GitLab administrator may want to disable Terraform state to reduce diskspace or if Terraform is not used in your instance. +A GitLab administrator may want to disable Terraform state to reduce disk space or if Terraform is not used in your instance. To do so, follow the steps below according to your installation's type. **In Omnibus installations:** diff --git a/doc/administration/timezone.md b/doc/administration/timezone.md index 6f460ed0ea8..bc94110c5b6 100644 --- a/doc/administration/timezone.md +++ b/doc/administration/timezone.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Changing your time zone +# Changing your time zone **(FREE SELF)** The global time zone configuration parameter can be changed in `config/gitlab.yml`: diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 5a8ee1c5c94..6861cdcde4e 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Debugging Tips +# Debugging tips **(FREE SELF)** Sometimes things don't work the way they should. Here are some tips on debugging issues out in production. @@ -26,7 +26,7 @@ You can enable output of Active Record debug logging in the Rails console session by running: ```ruby -ActiveRecord::Base.logger = Logger.new(STDOUT) +ActiveRecord::Base.logger = Logger.new($stdout) ``` This will show information about database queries triggered by any Ruby code @@ -155,7 +155,7 @@ and more. However, this is not enabled by default. To enable it, define the gitlab_rails['env'] = {"ENABLE_RBTRACE" => "1"} ``` -Then reconfigure the system and restart Unicorn and Sidekiq. To run this +Then reconfigure the system and restart Puma and Sidekiq. To run this in Omnibus, run as root: ```ruby @@ -178,7 +178,7 @@ following tips are only recommended if you do NOT mind users being affected by downtime. Otherwise skip to the next section. 1. Load the problematic URL -1. Run `sudo gdb -p ` to attach to the Unicorn process. +1. Run `sudo gdb -p ` to attach to the Puma process. 1. In the GDB window, type: ```plaintext @@ -186,7 +186,7 @@ downtime. Otherwise skip to the next section. ``` 1. This forces the process to generate a Ruby backtrace. Check - `/var/log/gitlab/unicorn/unicorn_stderr.log` for the backtrace. For example, you may see: + `/var/log/gitlab/puma/puma_stderr.log` for the backtrace. For example, you may see: ```plaintext from /opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/sampler.rb:33:in `block in start' @@ -213,16 +213,19 @@ downtime. Otherwise skip to the next section. exit ``` -Note that if the Unicorn process terminates before you are able to run these +Note that if the Puma process terminates before you are able to run these commands, GDB will report an error. To buy more time, you can always raise the -Unicorn timeout. For omnibus users, you can edit `/etc/gitlab/gitlab.rb` and -increase it from 60 seconds to 300: +Puma worker timeout. For omnibus users, you can edit `/etc/gitlab/gitlab.rb` and +increase it from 60 seconds to 600: ```ruby -unicorn['worker_timeout'] = 300 +gitlab_rails['env'] = { + 'GITLAB_RAILS_RACK_TIMEOUT' => 600 +} ``` -For source installations, edit `config/unicorn.rb`. +For source installations, set the environment variable. +Refer to [Puma Worker timeout](https://docs.gitlab.com/omnibus/settings/puma.html#worker-timeout). [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -267,7 +270,7 @@ is a Unicorn worker that is spinning via `top`. Try to use the `gdb` techniques above. In addition, using `strace` may help isolate issues: ```shell -strace -ttTfyyy -s 1024 -p -o /tmp/unicorn.txt +strace -ttTfyyy -s 1024 -p -o /tmp/puma.txt ``` If you cannot isolate which Unicorn worker is the issue, try to run `strace` @@ -275,10 +278,10 @@ on all the Unicorn workers to see where the [`/internal/allowed`](../../development/internal_api.md) endpoint gets stuck: ```shell -ps auwx | grep unicorn | awk '{ print " -p " $2}' | xargs strace -ttTfyyy -s 1024 -o /tmp/unicorn.txt +ps auwx | grep puma | awk '{ print " -p " $2}' | xargs strace -ttTfyyy -s 1024 -o /tmp/puma.txt ``` -The output in `/tmp/unicorn.txt` may help diagnose the root cause. +The output in `/tmp/puma.txt` may help diagnose the root cause. ## More information diff --git a/doc/administration/troubleshooting/defcon.md b/doc/administration/troubleshooting/defcon.md index 09e11553d97..7cae6ea1c8f 100644 --- a/doc/administration/troubleshooting/defcon.md +++ b/doc/administration/troubleshooting/defcon.md @@ -5,21 +5,21 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Disaster Recovery +# Disaster recovery **(FREE SELF)** -This document describes a feature that allows to easily disable some important but computationally -expensive parts of the application, in order to relieve stress on the database in an ongoing downtime. +This document describes a feature that allows you to disable some important but computationally +expensive parts of the application to relieve stress on the database during an ongoing downtime. ## `ci_queueing_disaster_recovery` -This feature flag, if enabled temporarily disables fair scheduling on shared runners. -This can help reduce system resource usage on the `jobs/request` endpoint -by significantly reducing computations being performed. +This feature flag, if temporarily enabled, disables fair scheduling on shared runners. +This can help to reduce system resource usage on the `jobs/request` endpoint +by significantly reducing the computations being performed. Side effects: -- In case of a large backlog of jobs, the jobs will be processed in the order -they were put in the system instead of balancing the jobs across many projects +- In case of a large backlog of jobs, the jobs are processed in the order + they were put in the system, instead of balancing the jobs across many projects. - Projects which are out of quota will be run. This affects -only jobs that were created during the last hour, as prior jobs are canceled -by a periodic background worker (`StuckCiJobsWorker`). + only jobs created during the last hour, as prior jobs are canceled + by a periodic background worker (`StuckCiJobsWorker`). diff --git a/doc/administration/troubleshooting/diagnostics_tools.md b/doc/administration/troubleshooting/diagnostics_tools.md index 27a7493b318..fe85b5d5803 100644 --- a/doc/administration/troubleshooting/diagnostics_tools.md +++ b/doc/administration/troubleshooting/diagnostics_tools.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Diagnostics tools +# Diagnostics tools **(FREE SELF)** These are some of the diagnostics tools the GitLab Support team uses during troubleshooting. They are listed here for transparency, and they may be useful for users with experience diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 588be73e786..92070a86a0d 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -100,7 +100,7 @@ Rails.cache.instance_variable_get(:@data).keys ```ruby # Before 11.6.0 -logger = Logger.new(STDOUT) +logger = Logger.new($stdout) admin_token = User.find_by_username('ADMIN_USERNAME').personal_access_tokens.first.token app.get("URL/?private_token=#{admin_token}") @@ -113,7 +113,7 @@ Gitlab::Profiler.with_user(admin) { app.get(url) } ## Using the GitLab profiler inside console (used as of 10.5) ```ruby -logger = Logger.new(STDOUT) +logger = Logger.new($stdout) admin = User.find_by_username('ADMIN_USERNAME') Gitlab::Profiler.profile('URL', logger: logger, user: admin) ``` @@ -279,6 +279,21 @@ p.each do |project| end ``` +## Bulk update to change all the Jira integrations to Jira instance-level values + +To change all Jira project to use the instance-level integration settings: + +1. In a Rails console: + + ```ruby + jira_service_instance_id = JiraService.find_by(instance: true).id + JiraService.where(active: true, instance: false, template: false, inherit_from_id: nil).find_each do |service| + service.update_attribute(:inherit_from_id, jira_service_instance_id) + end + ``` + +1. Modify and save again the instance-level integration from the UI to propagate the changes to all the group-level and project-level integrations. + ### Bulk update to disable the Slack Notification service To disable notifications for all projects that have Slack service enabled, do: @@ -302,7 +317,18 @@ the displayed size may still show old sizes or commit numbers. To force an updat p = Project.find_by_full_path('/') pp p.statistics p.statistics.refresh! -pp p.statistics # compare with earlier values +pp p.statistics +# compare with earlier values + +# check the total artifact storage space separately +builds_with_artifacts = p.builds.with_downloadable_artifacts.all + +artifact_storage = 0 +builds_with_artifacts.find_each do |build| + artifact_storage += build.artifacts_size +end + +puts "#{artifact_storage} bytes" ``` ### Identify deploy keys associated with blocked and non-member users @@ -341,6 +367,16 @@ DeployKeysProject.with_write_access.find_each do |deploy_key_mapping| end ``` +### Find projects using an SQL query + +Find and store an array of projects based on an SQL query: + +```ruby +# Finds projects that end with '%ject' +projects = Project.find_by_sql("SELECT * FROM projects WHERE name LIKE '%ject'") +=> [#>, #>] +``` + ## Wikis ### Recreate @@ -524,7 +560,8 @@ User.billable.count Using cURL and jq (up to a max 100, see the [pagination docs](../../api/README.md#pagination)): ```shell -curl --silent --header "Private-Token: ********************" "https://gitlab.example.com/api/v4/users?per_page=100&active" | jq --compact-output '.[] | [.id,.name,.username]' +curl --silent --header "Private-Token: ********************" \ + "https://gitlab.example.com/api/v4/users?per_page=100&active" | jq --compact-output '.[] | [.id,.name,.username]' ``` ### Block or Delete Users that have no projects or groups @@ -694,6 +731,16 @@ emails.each do |e| end ``` +### Find groups using an SQL query + +Find and store an array of groups based on an SQL query: + +```ruby +# Finds groups and subgroups that end with '%oup' +Group.find_by_sql("SELECT * FROM namespaces WHERE name LIKE '%oup'") +=> [#, #] +``` + ## Routes ### Remove redirecting routes @@ -839,7 +886,7 @@ License.current.trial? ### Check if a project feature is available on the instance -Features listed in . +Features listed in . ```ruby License.current.feature_available?(:jira_dev_panel_integration) @@ -847,7 +894,7 @@ License.current.feature_available?(:jira_dev_panel_integration) ### Check if a project feature is available in a project -Features listed in [`license.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/license.rb). +Features listed in [`license.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/license.rb). ```ruby p = Project.find_by_full_path('/') @@ -863,55 +910,6 @@ license.save License.current # check to make sure it applied ``` -## Unicorn - -From [Zendesk ticket #91083](https://gitlab.zendesk.com/agent/tickets/91083) (internal) - -### Poll Unicorn requests by seconds - -```ruby -require 'rubygems' -require 'unicorn' - -# Usage for this program -def usage - puts "ruby unicorn_status.rb " - puts "Polls the given Unix socket every interval in seconds. Will not allow you to drop below 3 second poll intervals." - puts "Example: /opt/gitlab/embedded/bin/ruby poll_unicorn.rb /var/opt/gitlab/gitlab-rails/sockets/gitlab.socket 10" -end - -# Look for required args. Throw usage and exit if they don't exist. -if ARGV.count < 2 - usage - exit 1 -end - -# Get the socket and threshold values. -socket = ARGV[0] -threshold = (ARGV[1]).to_i - -# Check threshold - is it less than 3? If so, set to 3 seconds. Safety first! -if threshold.to_i < 3 - threshold = 3 -end - -# Check - does that socket exist? -unless File.exist?(socket) - puts "Socket file not found: #{socket}" - exit 1 -end - -# Poll the given socket every THRESHOLD seconds as specified above. -puts "Running infinite loop. Use CTRL+C to exit." -puts "------------------------------------------" -loop do - Raindrops::Linux.unix_listener_stats([socket]).each do |addr, stats| - puts DateTime.now.to_s + " Active: " + stats.active.to_s + " Queued: " + stats.queued.to_s - end - sleep threshold -end -``` - ## Registry ### Registry Disk Space Usage by Project @@ -1189,6 +1187,28 @@ Prints the metrics saved in `conversational_development_index_metrics`. rake gitlab:usage_data:generate_and_send ``` +## Kubernetes integration + +Find cluster: + +```ruby +cluster = Clusters::Cluster.find(1) +cluster = Clusters::Cluster.find_by(name: 'cluster_name') +``` + +Delete cluster without associated resources: + +```ruby +# Find an admin user +user = User.find_by(username: 'admin_user') + +# Find the cluster with the ID +cluster = Clusters::Cluster.find(1) + +# Delete the cluster +Clusters::DestroyService.new(user).execute(cluster) +``` + ## Elasticsearch ### Configuration attributes @@ -1206,11 +1226,11 @@ Among other attributes, in the output you will notice that all the settings avai You can then set anyone of Elasticsearch integration settings by issuing a command similar to: ```ruby -ApplicationSetting.last.update_attributes(elasticsearch_url: '') +ApplicationSetting.last.update(elasticsearch_url: '') #or -ApplicationSetting.last.update_attributes(elasticsearch_indexing: false) +ApplicationSetting.last.update(elasticsearch_indexing: false) ``` #### Getting attributes diff --git a/doc/administration/troubleshooting/group_saml_scim.md b/doc/administration/troubleshooting/group_saml_scim.md index 63e69589b54..9e9ef492ebd 100644 --- a/doc/administration/troubleshooting/group_saml_scim.md +++ b/doc/administration/troubleshooting/group_saml_scim.md @@ -42,6 +42,10 @@ SCIM mapping: ![Azure AD SCIM](img/AzureAD-scim_attribute_mapping.png) +Group Sync: + +![Azure Group Claims](img/azure_configure_group_claim.png) + ## Okta Basic SAML app configuration: diff --git a/doc/administration/troubleshooting/img/azure_configure_group_claim.png b/doc/administration/troubleshooting/img/azure_configure_group_claim.png new file mode 100644 index 00000000000..31df5fff625 Binary files /dev/null and b/doc/administration/troubleshooting/img/azure_configure_group_claim.png differ diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md index 1c205cc987a..75c5ce460e9 100644 --- a/doc/administration/troubleshooting/index.md +++ b/doc/administration/troubleshooting/index.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Troubleshooting a GitLab installation +# Troubleshooting a GitLab installation **(FREE SELF)** This page documents a collection of resources to help you troubleshoot a GitLab installation. diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 9766b2210ca..b43825092b7 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Kubernetes, GitLab and You +# Kubernetes, GitLab, and you **(FREE SELF)** This is a list of useful information regarding Kubernetes that the GitLab Support Team sometimes uses while troubleshooting. GitLab is making this public, so that anyone @@ -147,7 +147,7 @@ and they will assist you with any issues you are having. You can also use `gitlab-rake`, instead of `/usr/local/bin/gitlab-rake`. -- Troubleshooting **Operations > Kubernetes** integration: +- Troubleshooting **Infrastructure > Kubernetes** integration: - Check the output of `kubectl get events -w --all-namespaces`. - Check the logs of pods within `gitlab-managed-apps` namespace. diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index c4e991ccc1b..9eadbad171e 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Linux Cheat Sheet +# Linux cheat sheet **(FREE SELF)** This is the GitLab Support Team's collection of information regarding Linux, that they sometimes use while troubleshooting. It is listed here for transparency, @@ -177,8 +177,8 @@ strace -tt -T -f -y -yy -s 1024 -p # -o output file -# run strace on all unicorn processes -ps auwx | grep unicorn | awk '{ print " -p " $2}' | xargs strace -tt -T -f -y -yy -s 1024 -o /tmp/unicorn.txt +# run strace on all puma processes +ps auwx | grep puma | awk '{ print " -p " $2}' | xargs strace -tt -T -f -y -yy -s 1024 -o /tmp/puma.txt ``` Be aware that strace can have major impacts to system performance when it is running. diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index a0f71960e14..c16302dd251 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Parsing GitLab logs with `jq` +# Parsing GitLab logs with `jq` **(FREE SELF)** We recommend using log aggregation and search tools like Kibana and Splunk whenever possible, but if they are not available you can still quickly parse diff --git a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md index 481c95b925a..e55118d7309 100644 --- a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md +++ b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Navigating GitLab via Rails console +# Navigating GitLab via Rails console **(FREE SELF)** At the heart of GitLab is a web application [built using the Ruby on Rails framework](https://about.gitlab.com/blog/2018/10/29/why-we-use-rails-to-build-gitlab/). @@ -46,7 +46,7 @@ Let's enable debug logging for Active Record so we can see the underlying database queries made: ```ruby -ActiveRecord::Base.logger = Logger.new(STDOUT) +ActiveRecord::Base.logger = Logger.new($stdout) ``` Now, let's try retrieving a user from the database: diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 9565b7594d6..341c6bfbc65 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -53,8 +53,7 @@ This section is for links to information elsewhere in the GitLab documentation. - [PostgreSQL scaling](../postgresql/replication_and_failover.md) - Including [troubleshooting](../postgresql/replication_and_failover.md#troubleshooting) - `gitlab-ctl repmgr-check-master` (or `gitlab-ctl patroni check-leader` if - you're using Patroni) and PgBouncer errors. + `gitlab-ctl patroni check-leader` and PgBouncer errors. - [Developer database documentation](../../development/README.md#database-guides), some of which is absolutely not for production use. Including: diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 297a8355036..7a8ac8c3dbe 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Troubleshooting Sidekiq +# Troubleshooting Sidekiq **(FREE SELF)** Sidekiq is the background job processor GitLab uses to asynchronously run tasks. When things go wrong it can be difficult to troubleshoot. These diff --git a/doc/administration/troubleshooting/ssl.md b/doc/administration/troubleshooting/ssl.md index 7c0b745f8c2..5ce4c7f0fb2 100644 --- a/doc/administration/troubleshooting/ssl.md +++ b/doc/administration/troubleshooting/ssl.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Troubleshooting SSL +# Troubleshooting SSL **(FREE SELF)** This page contains a list of common SSL-related errors and scenarios that you may encounter while working with GitLab. It should serve as an addition to the diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md index 16ef4f83978..5fe493a536c 100644 --- a/doc/administration/troubleshooting/test_environments.md +++ b/doc/administration/troubleshooting/test_environments.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Apps for a Testing Environment +# Apps for a testing environment **(FREE SELF)** This is the GitLab Support Team's collection of information regarding testing environments, for use while troubleshooting. It is listed here for transparency, and it may be useful diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md index ad2b8586b8b..7b9ce5c6d7b 100644 --- a/doc/administration/troubleshooting/tracing_correlation_id.md +++ b/doc/administration/troubleshooting/tracing_correlation_id.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Finding relevant log entries with a correlation ID +# Finding relevant log entries with a correlation ID **(FREE SELF)** In GitLab 11.6 and later, a unique request tracking ID, known as the "correlation ID" has been logged by the GitLab instance for most requests. Each individual request to GitLab gets diff --git a/doc/administration/whats-new.md b/doc/administration/whats-new.md index a5e3a232890..ae19e0f0341 100644 --- a/doc/administration/whats-new.md +++ b/doc/administration/whats-new.md @@ -29,7 +29,8 @@ in the first patch release, such as `13.10.1`. You can configure the What's new variant: -1. Navigate to **Admin Area > Settings > Preferences**, then expand **What's new**. +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. On the left sidebar, select **Settings > Preferences**, then expand **What's new**. 1. Choose one of the following options: | Option | Description | diff --git a/doc/api/README.md b/doc/api/README.md index db0c593dc03..c23a383c70f 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Use the GitLab [REST](http://spec.openapis.org/oas/v3.0.3) API to automate GitLab. -You can also use a partial [OpenAPI definition](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/api/openapi/openapi.yaml), +You can also use a partial [OpenAPI definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/api/openapi/openapi.yaml), to test the API directly from the GitLab user interface. Contributions are welcome. @@ -59,8 +59,8 @@ version number. New features and bug fixes are released in tandem with GitLab. Apart from incidental patch and security releases, GitLab is released on the 22nd of each -month. Backward-incompatible changes (for example, endpoint and parameter removal), -and removal of entire API versions are done in tandem with major GitLab releases. +month. Major API version changes, and removal of entire API versions, are done in tandem +with major GitLab releases. All deprecations and changes between versions are in the documentation. For the changes between v3 and v4, see the [v3 to v4 documentation](v3_to_v4.md). @@ -73,7 +73,7 @@ Only API version v4 is available. Version v3 was removed in ## How to use the API API requests must include both `api` and the API version. The API -version is defined in [`lib/api.rb`](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/api/api.rb). +version is defined in [`lib/api.rb`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/api/api.rb). For example, the root of the v4 API is at `/api/v4`. ### Valid API request @@ -597,7 +597,8 @@ send the payload body: - Request payload (JSON): ```shell - curl --request POST --header "Content-Type: application/json" --data '{"name":"", "description":"", "description":"" --data "name=MyApplication&redirect_uri=http://redirect.uri&scopes=" "https://gitlab.example.com/api/v4/applications" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "name=MyApplication&redirect_uri=http://redirect.uri&scopes=" \ + "https://gitlab.example.com/api/v4/applications" ``` Example response: diff --git a/doc/api/audit_events.md b/doc/api/audit_events.md index 93fefe50d9e..dec2b85c61d 100644 --- a/doc/api/audit_events.md +++ b/doc/api/audit_events.md @@ -11,6 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Instance Audit Events **(PREMIUM SELF)** The Audit Events API allows you to retrieve [instance audit events](../administration/audit_events.md#instance-events). +This API cannot retrieve group or project audit events. To retrieve audit events using the API, you must [authenticate yourself](README.md#authentication) as an Administrator. @@ -133,6 +134,7 @@ Example response: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34078) in GitLab 12.5. The Group Audit Events API allows you to retrieve [group audit events](../administration/audit_events.md#group-events). +This API cannot retrieve project audit events. A user with a Owner role (or above) can retrieve group audit events of all users. A user with a Developer or Maintainer role is limited to group audit events based on their individual actions. diff --git a/doc/api/boards.md b/doc/api/boards.md index 3252036c840..3cdd9552d66 100644 --- a/doc/api/boards.md +++ b/doc/api/boards.md @@ -250,7 +250,8 @@ Example response: "path_with_namespace": "diaspora/diaspora-project-site", "created_at": "2018-07-03T05:48:49.982Z", "default_branch": null, - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "ssh_url_to_repo": "ssh://user@example.com/diaspora/diaspora-project-site.git", "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", diff --git a/doc/api/broadcast_messages.md b/doc/api/broadcast_messages.md index de56405056a..b98373b5a58 100644 --- a/doc/api/broadcast_messages.md +++ b/doc/api/broadcast_messages.md @@ -4,9 +4,7 @@ group: Activation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Broadcast Messages API - -> Introduced in GitLab 8.12. +# Broadcast Messages API **(FREE SELF)** Broadcast messages API operates on [broadcast messages](../user/admin_area/broadcast_messages.md). @@ -109,7 +107,9 @@ Parameters: Example request: ```shell -curl --data "message=Deploy in progress&color=#cecece" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/broadcast_messages" +curl --data "message=Deploy in progress&color=#cecece" \ + --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/broadcast_messages" ``` Example response: @@ -154,7 +154,8 @@ Parameters: Example request: ```shell -curl --request PUT --data "message=Update message&color=#000" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/broadcast_messages/1" +curl --request PUT --data "message=Update message&color=#000" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/broadcast_messages/1" ``` Example response: diff --git a/doc/api/commits.md b/doc/api/commits.md index 22d98b2b0a6..711b565bdbd 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -7,7 +7,7 @@ type: reference, api # Commits API **(FREE)** -This API operates on [repository commits](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository). Read more about [GitLab-specific information](../user/project/repository/index.md#commits) for commits. +This API operates on [repository commits](https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository). Read more about [GitLab-specific information](../user/project/repository/index.md#commit-changes-to-a-repository) for commits. ## List repository commits @@ -28,6 +28,7 @@ GET /projects/:id/repository/commits | `with_stats` | boolean | no | Stats about each commit are added to the response | | `first_parent` | boolean | no | Follow only the first parent commit upon seeing a merge commit | | `order` | string | no | List commits in order. Possible values: `default`, [`topo`](https://git-scm.com/docs/git-log#Documentation/git-log.txt---topo-order). Defaults to `default`, the commits are shown in reverse chronological order. | +| `trailers` | boolean | no | Parse and include [Git trailers](https://git-scm.com/docs/git-interpret-trailers) for every commit | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/repository/commits" @@ -141,7 +142,8 @@ PAYLOAD=$(cat << 'JSON' } JSON ) -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data "$PAYLOAD" "https://gitlab.example.com/api/v4/projects/1/repository/commits" +curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data "$PAYLOAD" "https://gitlab.example.com/api/v4/projects/1/repository/commits" ``` Example response: @@ -305,9 +307,11 @@ Parameters: | `sha` | string | yes | The commit hash | | `branch` | string | yes | The name of the branch | | `dry_run` | boolean | no | Does not commit any changes. Default is false. [Introduced in GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/231032) | +| `message` | string | no | A custom commit message to use for the new commit. [Introduced in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62481) ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "branch=master" "https://gitlab.example.com/api/v4/projects/5/repository/commits/master/cherry_pick" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "branch=master" "https://gitlab.example.com/api/v4/projects/5/repository/commits/master/cherry_pick" ``` Example response: @@ -380,7 +384,8 @@ Parameters: | `dry_run` | boolean | no | Does not commit any changes. Default is false. [Introduced in GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/231032) | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "branch=master" "https://gitlab.example.com/api/v4/projects/5/repository/commits/a738f717824ff53aebad8b090c1b79a14f2bd9e8/revert" +curl --request POST --header "PRIVATE-TOKEN: " --form "branch=master" \ + "https://gitlab.example.com/api/v4/projects/5/repository/commits/a738f717824ff53aebad8b090c1b79a14f2bd9e8/revert" ``` Example response: @@ -534,7 +539,9 @@ POST /projects/:id/repository/commits/:sha/comments | `line_type` | string | no | The line type. Takes `new` or `old` as arguments | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "note=Nice picture man\!" --form "path=dudeism.md" --form "line=11" --form "line_type=new" "https://gitlab.example.com/api/v4/projects/17/repository/commits/18f3e63d05582537db6d183d9d557be09e1f90c8/comments" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "note=Nice picture man\!" --form "path=dudeism.md" --form "line=11" --form "line_type=new" \ + "https://gitlab.example.com/api/v4/projects/17/repository/commits/18f3e63d05582537db6d183d9d557be09e1f90c8/comments" ``` Example response: @@ -791,6 +798,7 @@ Example response: "source_project_id":35, "target_project_id":35, "labels":[ ], + "draft":false, "work_in_progress":false, "milestone":null, "merge_when_pipeline_succeeds":false, diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index 0b37c0ad91a..4de024da2de 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -90,7 +90,8 @@ GET /groups/:id/registry/repositories | `tags_count` | boolean | no | If the parameter is included as true, each repository includes `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1&tags_count=true" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1&tags_count=true" ``` Example response: @@ -161,7 +162,8 @@ GET /registry/repositories/:id | `tags_count` | boolean | no | If the parameter is included as `true`, the response includes `"tags_count"`. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/registry/repositories/2?tags=true&tags_count=true" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/registry/repositories/2?tags=true&tags_count=true" ``` Example response: @@ -202,7 +204,8 @@ DELETE /projects/:id/registry/repositories/:repository_id | `repository_id` | integer | yes | The ID of registry repository. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2" +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2" ``` ## List registry repository tags @@ -221,7 +224,8 @@ GET /projects/:id/registry/repositories/:repository_id/tags | `repository_id` | integer | yes | The ID of registry repository. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` Example response: @@ -256,7 +260,8 @@ GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name | `tag_name` | string | yes | The name of tag. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0" ``` Example response: @@ -289,7 +294,8 @@ DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name | `tag_name` | string | yes | The name of tag. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0" +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0" ``` This action doesn't delete blobs. To delete them and recycle disk space, @@ -350,23 +356,27 @@ Examples: and remove ones that are older than 2 days: ```shell - curl --request DELETE --data 'name_regex_delete=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" + curl --request DELETE --data 'name_regex_delete=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` 1. Remove all tags, but keep always the latest 5: ```shell - curl --request DELETE --data 'name_regex_delete=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" + curl --request DELETE --data 'name_regex_delete=.*' --data 'keep_n=5' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` 1. Remove all tags, but keep always tags beginning with `stable`: ```shell - curl --request DELETE --data 'name_regex_delete=.*' --data 'name_regex_keep=stable.*' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" + curl --request DELETE --data 'name_regex_delete=.*' --data 'name_regex_keep=stable.*' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` 1. Remove all tags that are older than 1 month: ```shell - curl --request DELETE --data 'name_regex_delete=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" + curl --request DELETE --data 'name_regex_delete=.*' --data 'older_than=1month' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` diff --git a/doc/api/custom_attributes.md b/doc/api/custom_attributes.md index a17b3b78b18..56a9f6881cd 100644 --- a/doc/api/custom_attributes.md +++ b/doc/api/custom_attributes.md @@ -90,7 +90,8 @@ PUT /projects/:id/custom_attributes/:key | `value` | string | yes | The value of the custom attribute | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --data "value=Greenland" "https://gitlab.example.com/api/v4/users/42/custom_attributes/location" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --data "value=Greenland" "https://gitlab.example.com/api/v4/users/42/custom_attributes/location" ``` Example response: diff --git a/doc/api/dependency_proxy.md b/doc/api/dependency_proxy.md index 8448edef9c5..139669c018c 100644 --- a/doc/api/dependency_proxy.md +++ b/doc/api/dependency_proxy.md @@ -11,7 +11,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11631) in GitLab 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Free](https://about.gitlab.com/pricing/) in GitLab 13.6. -Deletes the cached manifests and blobs for a group. This endpoint requires group owner access. +Deletes the cached manifests and blobs for a group. This endpoint requires the [Owner role](../user/permissions.md) +for the group. WARNING: [A bug exists](https://gitlab.com/gitlab-org/gitlab/-/issues/277161) for this API. diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index 30a56e6253b..3b063180900 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -124,7 +124,9 @@ POST /projects/:id/deploy_keys | `can_push` | boolean | no | Can deploy key push to the project's repository | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"title": "My deploy key", "key": "ssh-rsa AAAA...", "can_push": "true"}' "https://gitlab.example.com/api/v4/projects/5/deploy_keys/" +curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"title": "My deploy key", "key": "ssh-rsa AAAA...", "can_push": "true"}' \ + "https://gitlab.example.com/api/v4/projects/5/deploy_keys/" ``` Example response: @@ -154,7 +156,8 @@ PUT /projects/:id/deploy_keys/:key_id | `can_push` | boolean | no | Can deploy key push to the project's repository | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"title": "New deploy key", "can_push": true}' "https://gitlab.example.com/api/v4/projects/5/deploy_keys/11" +curl --request PUT --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"title": "New deploy key", "can_push": true}' "https://gitlab.example.com/api/v4/projects/5/deploy_keys/11" ``` Example response: @@ -238,7 +241,9 @@ With those IDs, add the same deploy key to all: ```shell for project_id in 321 456 987; do - curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ - --data '{"title": "my key", "key": "ssh-rsa AAAA..."}' "https://gitlab.example.com/api/v4/projects/${project_id}/deploy_keys" + curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"title": "my key", "key": "ssh-rsa AAAA..."}' \ + "https://gitlab.example.com/api/v4/projects/${project_id}/deploy_keys" done ``` diff --git a/doc/api/deploy_tokens.md b/doc/api/deploy_tokens.md index ad144946445..d8aab8896a1 100644 --- a/doc/api/deploy_tokens.md +++ b/doc/api/deploy_tokens.md @@ -49,7 +49,8 @@ Example response: ## Project deploy tokens -Project deploy token API endpoints require project maintainer access or higher. +Project deploy token API endpoints require the [Maintainer role](../user/permissions.md) or higher +for the project. ### List project deploy tokens @@ -116,7 +117,9 @@ Parameters: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"name": "My deploy token", "expires_at": "2021-01-01", "username": "custom-user", "scopes": ["read_repository"]}' "https://gitlab.example.com/api/v4/projects/5/deploy_tokens/" +curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"name": "My deploy token", "expires_at": "2021-01-01", "username": "custom-user", "scopes": ["read_repository"]}' \ + "https://gitlab.example.com/api/v4/projects/5/deploy_tokens/" ``` Example response: @@ -156,7 +159,8 @@ Parameters: Example request: ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/deploy_tokens/13" +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/deploy_tokens/13" ``` ## Group deploy tokens @@ -229,7 +233,9 @@ Parameters: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"name": "My deploy token", "expires_at": "2021-01-01", "username": "custom-user", "scopes": ["read_repository"]}' "https://gitlab.example.com/api/v4/groups/5/deploy_tokens/" +curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"name": "My deploy token", "expires_at": "2021-01-01", "username": "custom-user", "scopes": ["read_repository"]}' \ + "https://gitlab.example.com/api/v4/groups/5/deploy_tokens/" ``` Example response: diff --git a/doc/api/deployments.md b/doc/api/deployments.md index cf224ad60ab..586f3edf51e 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -9,6 +9,9 @@ type: concepts, howto ## List project deployments +> The `updated_after` and `updated_before` attributes were removed and replaced + by `finished_after` and `finished_before` respectively in GitLab 14.0. + Get a list of deployments in a project. ```plaintext @@ -17,27 +20,19 @@ GET /projects/:id/deployments | Attribute | Type | Required | Description | |------------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `order_by` | string | no | Return deployments ordered by `id` or `iid` or `created_at` or `updated_at` or `ref` fields. Default is `id` | -| `sort` | string | no | Return deployments sorted in `asc` or `desc` order. Default is `asc` | -| `updated_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | -| `updated_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | -| `environment` | string | no | The [name of the environment](../ci/environments/index.md) to filter deployments by | -| `status` | string | no | The status to filter deployments by | - -The status attribute can be one of the following values: - -- created -- running -- success -- failed -- canceled +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `order_by` | string | no | Return deployments ordered by either one of `id`, `iid`, `created_at`, `updated_at` or `ref` fields. Default is `id`. | +| `sort` | string | no | Return deployments sorted in `asc` or `desc` order. Default is `asc`. | +| `finished_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `finished_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `environment` | string | no | The [name of the environment](../ci/environments/index.md) to filter deployments by. | +| `status` | string | no | The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`. ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments" ``` -Example of response +Example response: ```json [ @@ -51,16 +46,16 @@ Example of response "author_name": "Administrator", "created_at": "2016-08-11T09:36:01.000+02:00", "id": "99d03678b90d914dbb1b109132516d71a4a03ea8", - "message": "Merge branch 'new-title' into 'master'\r\n\r\nUpdate README\r\n\r\n\r\n\r\nSee merge request !1", + "message": "Merge branch 'new-title' into 'main'\r\n\r\nUpdate README\r\n\r\n\r\n\r\nSee merge request !1", "short_id": "99d03678", - "title": "Merge branch 'new-title' into 'master'\r" + "title": "Merge branch 'new-title' into 'main'\r" }, "coverage": null, "created_at": "2016-08-11T07:36:27.357Z", "finished_at": "2016-08-11T07:36:39.851Z", "id": 657, "name": "deploy", - "ref": "master", + "ref": "main", "runner": null, "stage": "deploy", "started_at": null, @@ -86,7 +81,7 @@ Example of response "pipeline": { "created_at": "2016-08-11T02:12:10.222Z", "id": 36, - "ref": "master", + "ref": "main", "sha": "99d03678b90d914dbb1b109132516d71a4a03ea8", "status": "success", "updated_at": "2016-08-11T02:12:10.222Z", @@ -100,7 +95,7 @@ Example of response }, "id": 41, "iid": 1, - "ref": "master", + "ref": "main", "sha": "99d03678b90d914dbb1b109132516d71a4a03ea8", "user": { "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", @@ -121,16 +116,16 @@ Example of response "author_name": "Administrator", "created_at": "2016-08-11T13:28:26.000+02:00", "id": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", - "message": "Merge branch 'rename-readme' into 'master'\r\n\r\nRename README\r\n\r\n\r\n\r\nSee merge request !2", + "message": "Merge branch 'rename-readme' into 'main'\r\n\r\nRename README\r\n\r\n\r\n\r\nSee merge request !2", "short_id": "a91957a8", - "title": "Merge branch 'rename-readme' into 'master'\r" + "title": "Merge branch 'rename-readme' into 'main'\r" }, "coverage": null, "created_at": "2016-08-11T11:32:24.456Z", "finished_at": "2016-08-11T11:32:35.145Z", "id": 664, "name": "deploy", - "ref": "master", + "ref": "main", "runner": null, "stage": "deploy", "started_at": null, @@ -156,7 +151,7 @@ Example of response "pipeline": { "created_at": "2016-08-11T07:43:52.143Z", "id": 37, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "status": "success", "updated_at": "2016-08-11T07:43:52.143Z", @@ -170,7 +165,7 @@ Example of response }, "id": 42, "iid": 2, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "user": { "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", @@ -199,13 +194,13 @@ GET /projects/:id/deployments/:deployment_id curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments/1" ``` -Example of response +Example response: ```json { "id": 42, "iid": 2, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "created_at": "2016-08-11T11:32:35.444Z", "updated_at": "2016-08-11T11:34:01.123Z", @@ -227,7 +222,7 @@ Example of response "status": "success", "stage": "deploy", "name": "deploy", - "ref": "master", + "ref": "main", "tag": false, "coverage": null, "created_at": "2016-08-11T11:32:24.456Z", @@ -252,16 +247,16 @@ Example of response "commit": { "id": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "short_id": "a91957a8", - "title": "Merge branch 'rename-readme' into 'master'\r", + "title": "Merge branch 'rename-readme' into 'main'\r", "author_name": "Administrator", "author_email": "admin@example.com", "created_at": "2016-08-11T13:28:26.000+02:00", - "message": "Merge branch 'rename-readme' into 'master'\r\n\r\nRename README\r\n\r\n\r\n\r\nSee merge request !2" + "message": "Merge branch 'rename-readme' into 'main'\r\n\r\nRename README\r\n\r\n\r\n\r\nSee merge request !2" }, "pipeline": { "created_at": "2016-08-11T07:43:52.143Z", "id": 42, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "status": "success", "updated_at": "2016-08-11T07:43:52.143Z", @@ -280,32 +275,25 @@ POST /projects/:id/deployments | Attribute | Type | Required | Description | |---------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `environment` | string | yes | The [name of the environment](../ci/environments/index.md) to create the deployment for | -| `sha` | string | yes | The SHA of the commit that is deployed | -| `ref` | string | yes | The name of the branch or tag that is deployed | -| `tag` | boolean | yes | A boolean that indicates if the deployed ref is a tag (true) or not (false) | -| `status` | string | yes | The status of the deployment | - -The status can be one of the following values: - -- created -- running -- success -- failed -- canceled +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user.| +| `environment` | string | yes | The [name of the environment](../ci/environments/index.md) to create the deployment for. | +| `sha` | string | yes | The SHA of the commit that is deployed. | +| `ref` | string | yes | The name of the branch or tag that is deployed. | +| `tag` | boolean | yes | A boolean that indicates if the deployed ref is a tag (`true`) or not (`false`). | +| `status` | string | no | The status to filter deployments by. One of `created`, `running`, `success`, `failed`, or `canceled`. | ```shell -curl --data "environment=production&sha=a91957a858320c0e17f3a0eca7cfacbff50ea29a&ref=master&tag=false&status=success" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments" +curl --data "environment=production&sha=a91957a858320c0e17f3a0eca7cfacbff50ea29a&ref=main&tag=false&status=success" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments" ``` -Example of a response: +Example response: ```json { "id": 42, "iid": 2, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "created_at": "2016-08-11T11:32:35.444Z", "status": "success", @@ -326,7 +314,7 @@ Example of a response: } ``` -## Updating a deployment +## Update a deployment ```plaintext PUT /projects/:id/deployments/:deployment_id @@ -334,21 +322,21 @@ PUT /projects/:id/deployments/:deployment_id | Attribute | Type | Required | Description | |------------------|----------------|----------|---------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `deployment_id` | integer | yes | The ID of the deployment to update | -| `status` | string | yes | The new status of the deployment | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | +| `deployment_id` | integer | yes | The ID of the deployment to update. | +| `status` | string | no | The new status of the deployment. One of `created`, `running`, `success`, `failed`, or `canceled`. | ```shell curl --request PUT --data "status=success" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments/42" ``` -Example of a response: +Example response: ```json { "id": 42, "iid": 2, - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "created_at": "2016-08-11T11:32:35.444Z", "status": "success", @@ -382,5 +370,5 @@ GET /projects/:id/deployments/:deployment_id/merge_requests It supports the same parameters as the [Merge Requests API](merge_requests.md#list-merge-requests) and returns a response using the same format: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments/42" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/deployments/42/merge_requests" ``` diff --git a/doc/api/discussions.md b/doc/api/discussions.md index 3d40349ecca..d8d989adfe2 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -156,7 +156,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `issue_iid` | integer | yes | The IID of an issue | | `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions?body=comment" @@ -167,7 +167,7 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/#start-a-thread-by-replying-to-a-standard-comment). **WARNING** -Notes can be added to other items than comments (system notes, etc.) making them threads. +Notes can be added to other items than comments, such as system notes, making them threads. ```plaintext POST /projects/:id/issues/:issue_iid/discussions/:discussion_id/notes @@ -182,7 +182,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a thread | | `note_id` | integer | yes | The ID of a thread note | | `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" @@ -365,7 +365,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `snippet_id` | integer | yes | The ID of an snippet | | `body` | string | yes | The content of a discussion | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions?body=comment" @@ -388,7 +388,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a thread | | `note_id` | integer | yes | The ID of a thread note | | `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" @@ -572,7 +572,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `epic_id` | integer | yes | The ID of an epic | | `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions?body=comment" @@ -596,7 +596,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a thread | | `note_id` | integer | yes | The ID of a thread note | | `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" @@ -847,7 +847,7 @@ Parameters for all comments: | `merge_request_iid` | integer | yes | The IID of a merge request | | `body` | string | yes | The content of the thread | | `commit_id` | string | no | SHA referencing commit to start this thread on | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | | `position` | hash | no | Position when creating a diff note | | `position[base_sha]` | string | yes | Base commit SHA in the source branch | | `position[start_sha]` | string | yes | SHA referencing commit in target branch | @@ -902,7 +902,7 @@ To create a new thread: "previous versions are here" ] ``` - + 1. Create a new diff thread. This example creates a thread on an added line: ```shell @@ -981,7 +981,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a thread | | `note_id` | integer | yes | The ID of a thread note | | `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment" @@ -1216,7 +1216,7 @@ Parameters: | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `commit_id` | integer | yes | The ID of a commit | | `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | | `position` | hash | no | Position when creating a diff note | | `position[base_sha]` | string | yes | Base commit SHA in the source branch | | `position[start_sha]` | string | yes | SHA referencing commit in target branch | @@ -1252,7 +1252,7 @@ Parameters: | `discussion_id` | integer | yes | The ID of a thread | | `note_id` | integer | yes | The ID of a thread note | | `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, e.g. 2016-03-11T03:45:40Z (requires administrator or project/group owner rights) | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/6a9c1750b37d513a43987b574953fceb50b03ce7/notes?body=comment diff --git a/doc/api/dora/metrics.md b/doc/api/dora/metrics.md index 31e6fee66ca..99826550b61 100644 --- a/doc/api/dora/metrics.md +++ b/doc/api/dora/metrics.md @@ -7,7 +7,8 @@ type: reference, api # DevOps Research and Assessment (DORA) key metrics API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/279039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/279039) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.10. +> - The legacy key/value pair `{ "" => "" }` was removed from the payload in GitLab 14.0. All methods require [reporter permissions and above](../../user/permissions.md). @@ -38,14 +39,14 @@ Example response: ```json [ - { "2021-03-01": 3, "date": "2021-03-01", "value": 3 }, - { "2021-03-02": 6, "date": "2021-03-02", "value": 6 }, - { "2021-03-03": 0, "date": "2021-03-03", "value": 0 }, - { "2021-03-04": 0, "date": "2021-03-04", "value": 0 }, - { "2021-03-05": 0, "date": "2021-03-05", "value": 0 }, - { "2021-03-06": 0, "date": "2021-03-06", "value": 0 }, - { "2021-03-07": 0, "date": "2021-03-07", "value": 0 }, - { "2021-03-08": 4, "date": "2021-03-08", "value": 4 } + { "date": "2021-03-01", "value": 3 }, + { "date": "2021-03-02", "value": 6 }, + { "date": "2021-03-03", "value": 0 }, + { "date": "2021-03-04", "value": 0 }, + { "date": "2021-03-05", "value": 0 }, + { "date": "2021-03-06", "value": 0 }, + { "date": "2021-03-07", "value": 0 }, + { "date": "2021-03-08", "value": 4 } ] ``` @@ -78,14 +79,14 @@ Example response: ```json [ - { "2021-03-01": 3, "date": "2021-03-01", "value": 3 }, - { "2021-03-02": 6, "date": "2021-03-02", "value": 6 }, - { "2021-03-03": 0, "date": "2021-03-03", "value": 0 }, - { "2021-03-04": 0, "date": "2021-03-04", "value": 0 }, - { "2021-03-05": 0, "date": "2021-03-05", "value": 0 }, - { "2021-03-06": 0, "date": "2021-03-06", "value": 0 }, - { "2021-03-07": 0, "date": "2021-03-07", "value": 0 }, - { "2021-03-08": 4, "date": "2021-03-08", "value": 4 } + { "date": "2021-03-01", "value": 3 }, + { "date": "2021-03-02", "value": 6 }, + { "date": "2021-03-03", "value": 0 }, + { "date": "2021-03-04", "value": 0 }, + { "date": "2021-03-05", "value": 0 }, + { "date": "2021-03-06", "value": 0 }, + { "date": "2021-03-07", "value": 0 }, + { "date": "2021-03-08", "value": 4 } ] ``` diff --git a/doc/api/dora4_group_analytics.md b/doc/api/dora4_group_analytics.md index 743dcf728a2..501bf824f03 100644 --- a/doc/api/dora4_group_analytics.md +++ b/doc/api/dora4_group_analytics.md @@ -1,5 +1,6 @@ --- redirect_to: 'dora/metrics.md' +remove_date: '2021-07-25' --- This document was moved to [another location](dora/metrics.md). diff --git a/doc/api/environments.md b/doc/api/environments.md index 7340fe9f9e9..4c93b3b15d5 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -164,7 +164,8 @@ POST /projects/:id/environments | `external_url` | string | no | Place to link to for this environment | ```shell -curl --data "name=deploy&external_url=https://deploy.gitlab.example.com" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/environments" +curl --data "name=deploy&external_url=https://deploy.gitlab.example.com" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/environments" ``` Example response: @@ -197,7 +198,8 @@ PUT /projects/:id/environments/:environments_id | `external_url` | string | no | The new `external_url` | ```shell -curl --request PUT --data "name=staging&external_url=https://staging.gitlab.example.com" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/environments/1" +curl --request PUT --data "name=staging&external_url=https://staging.gitlab.example.com" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/environments/1" ``` Example response: diff --git a/doc/api/feature_flag_specs.md b/doc/api/feature_flag_specs.md index 45db47f5ffe..33e454d50c4 100644 --- a/doc/api/feature_flag_specs.md +++ b/doc/api/feature_flag_specs.md @@ -8,293 +8,5 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.5. -WARNING: -This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). - -The API for creating, updating, reading and deleting Feature Flag Specs. -Automation engineers benefit from this API by being able to modify Feature Flag Specs without accessing user interface. -To manage the [Feature Flag](../operations/feature_flags.md) resources via public API, please refer to the [Feature Flags API](feature_flags.md) document. - -Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag Specs API. - -## List all effective feature flag specs under the specified environment - -Get all effective feature flag specs under the specified [environment](../ci/environments/index.md). - -For instance, there are two specs, `staging` and `production`, for a feature flag. -When you pass `production` as a parameter to this endpoint, the system returns -the `production` feature flag spec only. - -```plaintext -GET /projects/:id/feature_flag_scopes -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `environment` | string | yes | The [environment](../ci/environments/index.md) name | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/feature_flag_scopes?environment=production" -``` - -Example response: - -```json -[ - { - "id": 88, - "active": true, - "environment_scope": "production", - "strategies": [ - { - "name": "userWithId", - "parameters": { - "userIds": "1,2,3" - } - } - ], - "created_at": "2019-11-04T08:36:41.327Z", - "updated_at": "2019-11-04T08:36:41.327Z", - "name": "awesome_feature" - }, - { - "id": 82, - "active": true, - "environment_scope": "*", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:51.425Z", - "updated_at": "2019-11-04T08:39:45.751Z", - "name": "merge_train" - }, - { - "id": 81, - "active": false, - "environment_scope": "production", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.527Z", - "updated_at": "2019-11-04T08:13:10.527Z", - "name": "new_live_trace" - } -] -``` - -## List all specs of a feature flag - -Get all specs of a feature flag. - -```plaintext -GET /projects/:id/feature_flags/:name/scopes -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes" -``` - -Example response: - -```json -[ - { - "id": 79, - "active": false, - "environment_scope": "*", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.516Z", - "updated_at": "2019-11-04T08:13:10.516Z" - }, - { - "id": 80, - "active": true, - "environment_scope": "staging", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.525Z", - "updated_at": "2019-11-04T08:13:10.525Z" - }, - { - "id": 81, - "active": false, - "environment_scope": "production", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.527Z", - "updated_at": "2019-11-04T08:13:10.527Z" - } -] -``` - -## New feature flag spec - -Creates a new feature flag spec. - -```plaintext -POST /projects/:id/feature_flags/:name/scopes -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | -| `active` | boolean | yes | Whether the spec is active. | -| `strategies` | JSON | yes | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | - -```shell -curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes" \ - --header "PRIVATE-TOKEN: " \ - --header "Content-type: application/json" \ - --data @- << EOF -{ - "environment_scope": "*", - "active": false, - "strategies": [{ "name": "default", "parameters": {} }] -} -EOF -``` - -Example response: - -```json -{ - "id": 81, - "active": false, - "environment_scope": "*", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.527Z", - "updated_at": "2019-11-04T08:13:10.527Z" -} -``` - -## Single feature flag spec - -Gets a single feature flag spec. - -```plaintext -GET /projects/:id/feature_flags/:name/scopes/:environment_scope -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/:id/feature_flags/new_live_trace/scopes/production" -``` - -Example response: - -```json -{ - "id": 81, - "active": false, - "environment_scope": "production", - "strategies": [ - { - "name": "default", - "parameters": {} - } - ], - "created_at": "2019-11-04T08:13:10.527Z", - "updated_at": "2019-11-04T08:13:10.527Z" -} -``` - -## Edit feature flag spec - -Updates an existing feature flag spec. - -```plaintext -PUT /projects/:id/feature_flags/:name/scopes/:environment_scope -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | -| `active` | boolean | yes | Whether the spec is active. | -| `strategies` | JSON | yes | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | - -```shell -curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production" \ - --header "PRIVATE-TOKEN: " \ - --header "Content-type: application/json" \ - --data @- << EOF -{ - "active": true, - "strategies": [{ "name": "userWithId", "parameters": { "userIds": "1,2,3" } }] -} -EOF -``` - -Example response: - -```json -{ - "id": 81, - "active": true, - "environment_scope": "production", - "strategies": [ - { - "name": "userWithId", - "parameters": { "userIds": "1,2,3" } - } - ], - "created_at": "2019-11-04T08:13:10.527Z", - "updated_at": "2019-11-04T08:13:10.527Z" -} -``` - -## Delete feature flag spec - -Deletes a feature flag spec. - -```plaintext -DELETE /projects/:id/feature_flags/:name/scopes/:environment_scope -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `environment_scope` | string | yes | The URL-encoded [environment spec](../ci/environments/index.md#scoping-environments-with-specs) of the feature flag. | - -```shell -curl --header "PRIVATE-TOKEN: " --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace/scopes/production" -``` +This API was removed in [GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). +Please use [the new API](feature_flags.md) instead. diff --git a/doc/api/feature_flags_legacy.md b/doc/api/feature_flags_legacy.md index 6e0763b6015..262e1c537a4 100644 --- a/doc/api/feature_flags_legacy.md +++ b/doc/api/feature_flags_legacy.md @@ -9,315 +9,5 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in GitLab Premium 12.5. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to GitLab Free in 13.5. -WARNING: -This API is deprecated and [scheduled for removal in GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). Use [this API](feature_flags.md) instead. - -API for accessing resources of [GitLab Feature Flags](../operations/feature_flags.md). - -Users with Developer or higher [permissions](../user/permissions.md) can access Feature Flag API. - -## Feature Flags pagination - -By default, `GET` requests return 20 results at a time because the API results -are [paginated](README.md#pagination). - -## List feature flags for a project - -Gets all feature flags of the requested project. - -```plaintext -GET /projects/:id/feature_flags -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `scope` | string | no | The condition of feature flags, one of: `enabled`, `disabled`. | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/feature_flags" -``` - -Example response: - -```json -[ - { - "name":"merge_train", - "description":"This feature is about merge train", - "active": true, - "created_at":"2019-11-04T08:13:51.423Z", - "updated_at":"2019-11-04T08:13:51.423Z", - "scopes":[ - { - "id":82, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.425Z", - "updated_at":"2019-11-04T08:13:51.425Z" - }, - { - "id":83, - "active":true, - "environment_scope":"review/*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.427Z", - "updated_at":"2019-11-04T08:13:51.427Z" - }, - { - "id":84, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:51.428Z", - "updated_at":"2019-11-04T08:13:51.428Z" - } - ] - }, - { - "name":"new_live_trace", - "description":"This is a new live trace feature", - "active": true, - "created_at":"2019-11-04T08:13:10.507Z", - "updated_at":"2019-11-04T08:13:10.507Z", - "scopes":[ - { - "id":79, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.516Z", - "updated_at":"2019-11-04T08:13:10.516Z" - }, - { - "id":80, - "active":true, - "environment_scope":"staging", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.525Z", - "updated_at":"2019-11-04T08:13:10.525Z" - }, - { - "id":81, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.527Z", - "updated_at":"2019-11-04T08:13:10.527Z" - } - ] - } -] -``` - -## New feature flag - -Creates a new feature flag. - -```plaintext -POST /projects/:id/feature_flags -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | -| `description` | string | no | The description of the feature flag. | -| `active` | boolean | no | The active state of the flag. Defaults to true. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | -| `scopes` | JSON | no | The feature flag specs of the feature flag. | -| `scopes:environment_scope` | string | no | The environment spec. | -| `scopes:active` | boolean | no | Whether the spec is active. | -| `scopes:strategies` | JSON | no | The [strategies](../operations/feature_flags.md#feature-flag-strategies) of the feature flag spec. | - -```shell -curl "https://gitlab.example.com/api/v4/projects/1/feature_flags" \ - --header "PRIVATE-TOKEN: " \ - --header "Content-type: application/json" \ - --data @- << EOF -{ - "name": "awesome_feature", - "scopes": [{ "environment_scope": "*", "active": false, "strategies": [{ "name": "default", "parameters": {} }] }, - { "environment_scope": "production", "active": true, "strategies": [{ "name": "userWithId", "parameters": { "userIds": "1,2,3" } }] }] -} -EOF -``` - -Example response: - -```json -{ - "name":"awesome_feature", - "description":null, - "active": true, - "created_at":"2019-11-04T08:32:27.288Z", - "updated_at":"2019-11-04T08:32:27.288Z", - "scopes":[ - { - "id":85, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:32:29.324Z", - "updated_at":"2019-11-04T08:32:29.324Z" - }, - { - "id":86, - "active":true, - "environment_scope":"production", - "strategies":[ - { - "name":"userWithId", - "parameters":{ - "userIds":"1,2,3" - } - } - ], - "created_at":"2019-11-04T08:32:29.328Z", - "updated_at":"2019-11-04T08:32:29.328Z" - } - ] -} -``` - -## Single feature flag - -Gets a single feature flag. - -```plaintext -GET /projects/:id/feature_flags/:name -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/feature_flags/new_live_trace" -``` - -Example response: - -```json -{ - "name":"new_live_trace", - "description":"This is a new live trace feature", - "active": true, - "created_at":"2019-11-04T08:13:10.507Z", - "updated_at":"2019-11-04T08:13:10.507Z", - "scopes":[ - { - "id":79, - "active":false, - "environment_scope":"*", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.516Z", - "updated_at":"2019-11-04T08:13:10.516Z" - }, - { - "id":80, - "active":true, - "environment_scope":"staging", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.525Z", - "updated_at":"2019-11-04T08:13:10.525Z" - }, - { - "id":81, - "active":false, - "environment_scope":"production", - "strategies":[ - { - "name":"default", - "parameters":{ - - } - } - ], - "created_at":"2019-11-04T08:13:10.527Z", - "updated_at":"2019-11-04T08:13:10.527Z" - } - ] -} -``` - -## Delete feature flag - -Deletes a feature flag. - -```plaintext -DELETE /projects/:id/feature_flags/:name -``` - -| Attribute | Type | Required | Description | -| ------------------- | ---------------- | ---------- | ---------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `name` | string | yes | The name of the feature flag. | - -```shell -curl --header "PRIVATE-TOKEN: " --request DELETE "https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature" -``` +This API was removed in [GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). +Please use [the new API](feature_flags.md) instead. diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index b48ce48f6c3..85b36346167 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -4,12 +4,13 @@ group: Ecosystem info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Getting started with GitLab GraphQL API +# Get started with GitLab GraphQL API **(FREE)** This guide demonstrates basic usage of the GitLab GraphQL API. -See the [GraphQL API style guide](../../development/api_graphql_styleguide.md) for implementation details -aimed at developers who wish to work on developing the API itself. +Read the [GraphQL API style guide](../../development/api_graphql_styleguide.md) +for implementation details aimed at developers who wish to work on developing +the API itself. ## Running examples @@ -20,38 +21,43 @@ The examples documented here can be run using: ### Command line -You can run GraphQL queries in a `curl` request on the command line on your local machine. -A GraphQL request can be made as a `POST` request to `/api/graphql` with the query as the payload. -You can authorize your request by generating a [personal access token](../../user/profile/personal_access_tokens.md) -to use as a bearer token. +You can run GraphQL queries in a `curl` request on the command line on your +local computer. A GraphQL request can be made as a `POST` request to `/api/graphql` +with the query as the payload. You can authorize your request by generating a +[personal access token](../../user/profile/personal_access_tokens.md) to use as +a bearer token. Example: ```shell GRAPHQL_TOKEN= -curl "https://gitlab.com/api/graphql" --header "Authorization: Bearer $GRAPHQL_TOKEN" --header "Content-Type: application/json" --request POST --data "{\"query\": \"query {currentUser {name}}\"}" +curl "https://gitlab.com/api/graphql" --header "Authorization: Bearer $GRAPHQL_TOKEN" \ + --header "Content-Type: application/json" --request POST \ + --data "{\"query\": \"query {currentUser {name}}\"}" ``` ### GraphiQL -GraphiQL (pronounced "graphical") allows you to run queries directly against the server endpoint -with syntax highlighting and autocomplete. It also allows you to explore the schema and types. +GraphiQL (pronounced "graphical") allows you to run queries directly against +the server endpoint with syntax highlighting and autocomplete. It also allows +you to explore the schema and types. The examples below: -- Can be run directly against GitLab 11.0 or later, though some of the types and fields -may not be supported in older versions. -- Works against GitLab.com without any further setup. Make sure you are signed in and -navigate to the [GraphiQL Explorer](https://gitlab.com/-/graphql-explorer). +- Can be run directly against GitLab 11.0 or later, though some of the types + and fields may not be supported in older versions. +- Works against GitLab.com without any further setup. Make sure you are signed + in and navigate to the [GraphiQL Explorer](https://gitlab.com/-/graphql-explorer). -If you want to run the queries locally, or on a self-managed instance, -you must either: +If you want to run the queries locally, or on a self-managed instance, you must +either: -- Create the `gitlab-org` group with a project called `graphql-sandbox` under it. Create -several issues within the project. -- Edit the queries to replace `gitlab-org/graphql-sandbox` with your own group and project. +- Create the `gitlab-org` group with a project called `graphql-sandbox` under + it. Create several issues in the project. +- Edit the queries to replace `gitlab-org/graphql-sandbox` with your own group + and project. -Please refer to [running GraphiQL](index.md#graphiql) for more information. +Refer to [running GraphiQL](index.md#graphiql) for more information. NOTE: If you are running GitLab 11.0 to 12.0, enable the `graphql` @@ -72,8 +78,8 @@ which is an object identifier in the format of `"gid://gitlab/Issue/123"`. [GitLab GraphQL Schema](reference/index.md) outlines which objects and fields are available for clients to query and their corresponding data types. -Example: Get only the names of all the projects the currently logged in user can access (up to a limit, more on that later) -in the group `gitlab-org`. +Example: Get only the names of all the projects the currently logged in user can +access (up to a limit) in the group `gitlab-org`. ```graphql query { @@ -106,12 +112,12 @@ query { When retrieving child nodes use: -- the `edges { node { } }` syntax. -- the short form `nodes { }` syntax. +- The `edges { node { } }` syntax. +- The short form `nodes { }` syntax. Underneath it all is a graph we are traversing, hence the name GraphQL. -Example: Get a project (only its name) and the titles of all its issues. +Example: Get the name of a project, and the titles of all its issues. ```graphql query { @@ -128,23 +134,24 @@ query { ``` More about queries: -[GraphQL docs](https://graphql.org/learn/queries/) +[GraphQL documentation](https://graphql.org/learn/queries/) ### Authorization -Authorization uses the same engine as the GitLab application (and GitLab.com). So if you've signed in to GitLab -and use GraphiQL, all queries are performed as you, the signed in user. For more information, see the +Authorization uses the same engine as the GitLab application (and GitLab.com). +If you've signed in to GitLab and use GraphiQL, all queries are performed as +you, the signed in user. For more information, read the [GitLab API documentation](../README.md#authentication). ### Mutations -Mutations make changes to data. We can update, delete, or create new records. Mutations -generally use InputTypes and variables, neither of which appear here. +Mutations make changes to data. We can update, delete, or create new records. +Mutations generally use InputTypes and variables, neither of which appear here. Mutations have: - Inputs. For example, arguments, such as which emoji you'd like to award, -and to which object. + and to which object. - Return statements. That is, what you'd like to get back when it's successful. - Errors. Always ask for what went wrong, just in case. @@ -172,8 +179,9 @@ mutation { } ``` -Example: Add a comment to the issue (we're using the ID of the `GitLab.com` issue - but -if you're using a local instance, you must get the ID of an issue you can write to). +Example: Add a comment to the issue. In this example, we use the ID of the +`GitLab.com` issue. If you're using a local instance, you must get the ID of an +issue you can write to. ```graphql mutation { @@ -194,7 +202,8 @@ mutation { #### Update mutations -When you see the result `id` of the note you created - take a note of it. Now let's edit it to sip faster! +When you see the result `id` of the note you created, take a note of it. Let's +edit it to sip faster. ```graphql mutation { @@ -212,7 +221,7 @@ mutation { #### Deletion mutations -Let's delete the comment, since our tea is all gone. +Let's delete the comment, because our tea is all gone. ```graphql mutation { @@ -242,16 +251,18 @@ You should get something like the following output: We've asked for the note details, but it doesn't exist anymore, so we get `null`. More about mutations: -[GraphQL Docs](https://graphql.org/learn/queries/#mutations). +[GraphQL Documentation](https://graphql.org/learn/queries/#mutations). ### Introspective queries Clients can query the GraphQL endpoint for information about its own schema. by making an [introspective query](https://graphql.org/learn/introspection/). +The [GraphiQL Query Explorer](https://gitlab.com/-/graphql-explorer) uses an +introspection query to: -It is through an introspection query that the [GraphiQL Query Explorer](https://gitlab.com/-/graphql-explorer) -gets all of its knowledge about our GraphQL schema to do autocompletion and provide -its interactive `Docs` tab. +- Gain knowledge about our GraphQL schema. +- Do autocompletion. +- Provide its interactive `Docs` tab. Example: Get all the type names in the schema. @@ -265,8 +276,8 @@ Example: Get all the type names in the schema. } ``` -Example: Get all the fields associated with Issue. -`kind` tells us the enum value for the type, like `OBJECT`, `SCALAR` or `INTERFACE`. +Example: Get all the fields associated with Issue. `kind` tells us the enum +value for the type, like `OBJECT`, `SCALAR` or `INTERFACE`. ```graphql query IssueTypes { @@ -285,12 +296,12 @@ query IssueTypes { ``` More about introspection: -[GraphQL docs](https://graphql.org/learn/introspection/) +[GraphQL documentation](https://graphql.org/learn/introspection/) ## Sorting -Some of the GitLab GraphQL endpoints allow you to specify how you'd like a collection of -objects to be sorted. You can only sort by what the schema allows you to. +Some of the GitLab GraphQL endpoints allow you to specify how to sort a +collection of objects. You can only sort by what the schema allows you to. Example: Issues can be sorted by creation date: @@ -310,17 +321,18 @@ query { ## Pagination -Pagination is a way of only asking for a subset of the records (say, the first 10). -If we want more of them, we can make another request for the next 10 from the server -(in the form of something like "please give me the next 10 records"). +Pagination is a way of only asking for a subset of the records, such as the +first ten. If we want more of them, we can make another request for the next +ten from the server in the form of something like `please give me the next ten records`. -By default, the GitLab GraphQL API returns 100 records per page. -This can be changed by using `first` or `last` arguments. Both arguments take a value, -so `first: 10` returns the first 10 records, and `last: 10` the last 10 records. -There is a limit on how many records will be returned per page, which is generally `100`. +By default, the GitLab GraphQL API returns 100 records per page. To change this +behavior, use `first` or `last` arguments. Both arguments take a value, so +`first: 10` returns the first ten records, and `last: 10` the last ten records. +There is a limit on how many records are returned per page, which is generally +`100`. -Example: Retrieve only the first 2 issues (slicing). The `cursor` field gives us a position from which -we can retrieve further records relative to that one. +Example: Retrieve only the first two issues (slicing). The `cursor` field gives +us a position from which we can retrieve further records relative to that one. ```graphql query { @@ -341,9 +353,10 @@ query { } ``` -Example: Retrieve the next 3. (The cursor value +Example: Retrieve the next three. (The cursor value `eyJpZCI6IjI3MDM4OTMzIiwiY3JlYXRlZF9hdCI6IjIwMTktMTEtMTQgMDU6NTY6NDQgVVRDIn0` -could be different, but it's the `cursor` value returned for the second issue returned above.) +could be different, but it's the `cursor` value returned for the second issue +returned above.) ```graphql query { @@ -365,5 +378,5 @@ query { } ``` -More on pagination and cursors: -[GraphQL docs](https://graphql.org/learn/pagination/) +More about pagination and cursors: +[GraphQL documentation](https://graphql.org/learn/pagination/) diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 5864e5878b7..073f5faf57c 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -78,6 +78,11 @@ where the deprecated part of the schema is supported for a period of time before Clients should familiarize themselves with the process to avoid breaking changes affecting their integrations. +WARNING: +While GitLab will make all attempts to follow the [deprecation and removal process](#deprecation-and-removal-process), +GitLab may on very rare occasions need to make immediate breaking changes to the GraphQL API to patch critical security or performance +concerns and where the deprecation process would be considered to pose significant risk. + NOTE: Fields behind a feature flag and disabled by default are exempt from the deprecation process, and can be removed at any time without notice. @@ -129,7 +134,7 @@ New associations and root level objects are constantly being added. See the [GraphQL API Reference](reference/index.md) for up-to-date information. Root-level queries are defined in -[`app/graphql/types/query_type.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/graphql/types/query_type.rb). +[`app/graphql/types/query_type.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/types/query_type.rb). ### Multiplex queries diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 7f496e1aded..0069d109e1e 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -83,11 +83,11 @@ Fields related to design management. Returns [`DesignManagement!`](#designmanagement). -### `Query.devopsAdoptionSegments` +### `Query.devopsAdoptionEnabledNamespaces` -Get configured DevOps adoption segments on the instance. **BETA** This endpoint is subject to change without notice. +Get configured DevOps adoption namespaces. **BETA** This endpoint is subject to change without notice. -Returns [`DevopsAdoptionSegmentConnection`](#devopsadoptionsegmentconnection). +Returns [`DevopsAdoptionEnabledNamespaceConnection`](#devopsadoptionenablednamespaceconnection). This field returns a [connection](#connections). It accepts the four standard [pagination arguments](#connection-pagination-arguments): @@ -97,8 +97,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `directDescendantsOnly` | [`Boolean`](#boolean) | Limits segments to direct descendants of specified parent. | -| `parentNamespaceId` | [`NamespaceID`](#namespaceid) | Filter by ancestor namespace. | +| `displayNamespaceId` | [`NamespaceID`](#namespaceid) | Filter by display namespace. | ### `Query.echo` @@ -284,6 +283,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `search` | [`String`](#string) | Search query for project name, path, or description. | | `searchNamespaces` | [`Boolean`](#boolean) | Include namespace in project search. | | `sort` | [`String`](#string) | Sort order of results. | +| `topics` | [`[String!]`](#string) | Filters projects by topics. | ### `Query.runner` @@ -336,6 +336,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `search` | [`String`](#string) | Filter by full token or partial text in description field. | | `sort` | [`CiRunnerSort`](#cirunnersort) | Sort order of results. | | `status` | [`CiRunnerStatus`](#cirunnerstatus) | Filter runners by status. | | `tagList` | [`[String!]`](#string) | Filter by tags associated with the runner (comma-separated or array). | @@ -454,30 +455,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endDate` | [`ISO8601Date!`](#iso8601date) | Last day for which to fetch vulnerability history. | | `startDate` | [`ISO8601Date!`](#iso8601date) | First day for which to fetch vulnerability history. | -### `Query.vulnerabilitiesCountByDayAndSeverity` - -Number of vulnerabilities per severity level, per day, for the projects on the -current user's instance security dashboard. -. - -WARNING: -**Deprecated** in 13.3. -Use of this is not recommended. -Use: [`Query.vulnerabilitiesCountByDay`](#queryvulnerabilitiescountbyday). - -Returns [`VulnerabilitiesCountByDayAndSeverityConnection`](#vulnerabilitiescountbydayandseverityconnection). - -This field returns a [connection](#connections). It accepts the -four standard [pagination arguments](#connection-pagination-arguments): -`before: String`, `after: String`, `first: Int`, `last: Int`. - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `endDate` | [`ISO8601Date!`](#iso8601date) | Last day for which to fetch vulnerability history. | -| `startDate` | [`ISO8601Date!`](#iso8601date) | First day for which to fetch vulnerability history. | - ### `Query.vulnerability` Find a vulnerability. @@ -509,30 +486,6 @@ mutation($id: NoteableID!, $body: String!) { } ``` -### `Mutation.addAwardEmoji` - -WARNING: -**Deprecated** in 13.2. -Use awardEmojiAdd. - -Input type: `AddAwardEmojiInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardableId` | [`AwardableID!`](#awardableid) | The global ID of the awardable resource. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `name` | [`String!`](#string) | The emoji name. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardEmoji` | [`AwardEmoji`](#awardemoji) | The award emoji after mutation. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | - ### `Mutation.addProjectToSecurityDashboard` Input type: `AddProjectToSecurityDashboardInput` @@ -713,6 +666,28 @@ Input type: `AwardEmojiToggleInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `toggledOn` | [`Boolean!`](#boolean) | Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji. | +### `Mutation.boardEpicCreate` + +Input type: `BoardEpicCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `boardId` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Global ID of the board that the epic is in. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `groupPath` | [`ID!`](#id) | Group the epic to create is in. | +| `listId` | [`BoardsEpicListID!`](#boardsepiclistid) | Global ID of the epic board list in which epic will be created. | +| `title` | [`String!`](#string) | Title of the epic. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `epic` | [`Epic`](#epic) | Epic after creation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.boardListCreate` Input type: `BoardListCreateInput` @@ -759,26 +734,27 @@ Input type: `BoardListUpdateLimitMetricsInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `list` | [`BoardList`](#boardlist) | The updated list. | -### `Mutation.bulkFindOrCreateDevopsAdoptionSegments` +### `Mutation.bulkEnableDevopsAdoptionNamespaces` **BETA** This endpoint is subject to change without notice. -Input type: `BulkFindOrCreateDevopsAdoptionSegmentsInput` +Input type: `BulkEnableDevopsAdoptionNamespacesInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `namespaceIds` | [`[NamespaceID!]!`](#namespaceid) | List of Namespace IDs for the segments. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `displayNamespaceId` | [`NamespaceID`](#namespaceid) | Display namespace ID. | +| `namespaceIds` | [`[NamespaceID!]!`](#namespaceid) | List of Namespace IDs. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `segments` | [`[DevopsAdoptionSegment!]`](#devopsadoptionsegment) | Created segments after mutation. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `enabledNamespaces` | [`[DevopsAdoptionEnabledNamespace!]`](#devopsadoptionenablednamespace) | Enabled namespaces after mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | ### `Mutation.ciCdSettingsUpdate` @@ -790,6 +766,7 @@ Input type: `CiCdSettingsUpdateInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | +| `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI job tokens generated in this project have restricted access to resources. | | `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for this project. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | @@ -882,6 +859,7 @@ Input type: `CommitCreateInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `commit` | [`Commit`](#commit) | The commit after mutation. | | `commitPipelinePath` | [`String`](#string) | ETag path for the commit's pipeline. | +| `content` | [`[String!]`](#string) | Contents of the commit. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | ### `Mutation.configureSast` @@ -1093,27 +1071,6 @@ Input type: `CreateCustomEmojiInput` | `customEmoji` | [`CustomEmoji`](#customemoji) | The new custom emoji. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -### `Mutation.createDevopsAdoptionSegment` - -**BETA** This endpoint is subject to change without notice. - -Input type: `CreateDevopsAdoptionSegmentInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `namespaceId` | [`NamespaceID!`](#namespaceid) | Namespace ID to set for the segment. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `segment` | [`DevopsAdoptionSegment`](#devopsadoptionsegment) | The segment after mutation. | - ### `Mutation.createDiffNote` Input type: `CreateDiffNoteInput` @@ -1224,6 +1181,10 @@ Input type: `CreateIssueInput` ### `Mutation.createIteration` +WARNING: +**Deprecated** in 14.0. +Use iterationCreate. + Input type: `CreateIterationInput` #### Arguments @@ -1234,6 +1195,7 @@ Input type: `CreateIterationInput` | `description` | [`String`](#string) | The description of the iteration. | | `dueDate` | [`String`](#string) | The end date of the iteration. | | `groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | +| `iterationsCadenceId` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iterations cadence to be assigned to newly created iteration. | | `projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | | `startDate` | [`String`](#string) | The start date of the iteration. | | `title` | [`String`](#string) | The title of the iteration. | @@ -1476,7 +1438,6 @@ Input type: `DastScannerProfileCreateInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `globalId` **{warning-solid}** | [`DastScannerProfileID`](#dastscannerprofileid) | **Deprecated:** Use `id`. Deprecated in 13.6. | | `id` | [`DastScannerProfileID`](#dastscannerprofileid) | ID of the scanner profile. | ### `Mutation.dastScannerProfileDelete` @@ -1532,13 +1493,13 @@ Input type: `DastSiteProfileCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | -| `auth` | [`DastSiteProfileAuthInput`](#dastsiteprofileauthinput) | Parameters for authentication. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `auth` | [`DastSiteProfileAuthInput`](#dastsiteprofileauthinput) | Parameters for authentication. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. Defaults to `[]`. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. Defaults to `[]`. | | `fullPath` | [`ID!`](#id) | The project the site profile belongs to. | | `profileName` | [`String!`](#string) | The name of the site profile. | -| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | -| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. Will be ignored if `security_dast_site_profiles_api_option` feature flag is disabled. | +| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | +| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. | | `targetUrl` | [`String`](#string) | The URL of the target to be scanned. | #### Fields @@ -1576,14 +1537,14 @@ Input type: `DastSiteProfileUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | -| `auth` | [`DastSiteProfileAuthInput`](#dastsiteprofileauthinput) | Parameters for authentication. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `auth` | [`DastSiteProfileAuthInput`](#dastsiteprofileauthinput) | Parameters for authentication. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. | | `fullPath` | [`ID!`](#id) | The project the site profile belongs to. | | `id` | [`DastSiteProfileID!`](#dastsiteprofileid) | ID of the site profile to be updated. | | `profileName` | [`String!`](#string) | The name of the site profile. | -| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. Will be ignored if `security_dast_site_profiles_additional_fields` feature flag is disabled. | -| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. Will be ignored if `security_dast_site_profiles_api_option` feature flag is disabled. | +| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | +| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. | | `targetUrl` | [`String`](#string) | The URL of the target to be scanned. | #### Fields @@ -1676,26 +1637,6 @@ Input type: `DeleteAnnotationInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -### `Mutation.deleteDevopsAdoptionSegment` - -**BETA** This endpoint is subject to change without notice. - -Input type: `DeleteDevopsAdoptionSegmentInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `id` | [`[AnalyticsDevopsAdoptionSegmentID!]!`](#analyticsdevopsadoptionsegmentid) | One or many IDs of the segments to delete. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | - ### `Mutation.designManagementDelete` Input type: `DesignManagementDeleteInput` @@ -1912,6 +1853,26 @@ Input type: `DestroySnippetInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `snippet` | [`Snippet`](#snippet) | The snippet after mutation. | +### `Mutation.disableDevopsAdoptionNamespace` + +**BETA** This endpoint is subject to change without notice. + +Input type: `DisableDevopsAdoptionNamespaceInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`[AnalyticsDevopsAdoptionEnabledNamespaceID!]!`](#analyticsdevopsadoptionenablednamespaceid) | One or many IDs of the enabled namespaces to disable. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.discussionToggleResolve` Toggles the resolved state of a discussion. @@ -1934,30 +1895,27 @@ Input type: `DiscussionToggleResolveInput` | `discussion` | [`Discussion`](#discussion) | The discussion after mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -### `Mutation.dismissVulnerability` +### `Mutation.enableDevopsAdoptionNamespace` -WARNING: -**Deprecated** in 13.5. -Use vulnerabilityDismiss. +**BETA** This endpoint is subject to change without notice. -Input type: `DismissVulnerabilityInput` +Input type: `EnableDevopsAdoptionNamespaceInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `comment` | [`String`](#string) | Comment why vulnerability should be dismissed. | -| `dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason why vulnerability should be dismissed. | -| `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be dismissed. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `displayNamespaceId` | [`NamespaceID`](#namespaceid) | Display namespace ID. | +| `namespaceId` | [`NamespaceID!`](#namespaceid) | Namespace ID. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `vulnerability` | [`Vulnerability`](#vulnerability) | The vulnerability after dismissal. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `enabledNamespace` | [`DevopsAdoptionEnabledNamespace`](#devopsadoptionenablednamespace) | Enabled namespace after mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | ### `Mutation.environmentsCanaryIngressUpdate` @@ -2102,14 +2060,17 @@ Input type: `EpicMoveListInput` | `boardId` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Global ID of the board that the epic is in. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `epicId` | [`EpicID!`](#epicid) | ID of the epic to mutate. | -| `fromListId` | [`BoardsEpicListID!`](#boardsepiclistid) | ID of the board list that the epic will be moved from. | -| `toListId` | [`BoardsEpicListID!`](#boardsepiclistid) | ID of the board list that the epic will be moved to. | +| `fromListId` | [`BoardsEpicListID`](#boardsepiclistid) | ID of the board list that the epic will be moved from. Required if moving between lists. | +| `moveAfterId` | [`EpicID`](#epicid) | ID of epic that should be placed after the current epic. | +| `moveBeforeId` | [`EpicID`](#epicid) | ID of epic that should be placed before the current epic. | +| `toListId` | [`BoardsEpicListID!`](#boardsepiclistid) | ID of the list the epic will be in after mutation. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `epic` | [`Epic`](#epic) | The epic after mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | ### `Mutation.epicSetSubscription` @@ -2152,6 +2113,69 @@ Input type: `EpicTreeReorderInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.escalationPolicyCreate` + +Input type: `EscalationPolicyCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | The description of the escalation policy. | +| `name` | [`String!`](#string) | The name of the escalation policy. | +| `projectPath` | [`ID!`](#id) | The project to create the escalation policy for. | +| `rules` | [`[EscalationRuleInput!]!`](#escalationruleinput) | The steps of the escalation policy. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `escalationPolicy` | [`EscalationPolicyType`](#escalationpolicytype) | The escalation policy. | + +### `Mutation.escalationPolicyDestroy` + +Input type: `EscalationPolicyDestroyInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`IncidentManagementEscalationPolicyID!`](#incidentmanagementescalationpolicyid) | The escalation policy internal ID to remove. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `escalationPolicy` | [`EscalationPolicyType`](#escalationpolicytype) | The escalation policy. | + +### `Mutation.escalationPolicyUpdate` + +Input type: `EscalationPolicyUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | The description of the escalation policy. | +| `id` | [`IncidentManagementEscalationPolicyID!`](#incidentmanagementescalationpolicyid) | The ID of the on-call schedule to create the on-call rotation in. | +| `name` | [`String`](#string) | The name of the escalation policy. | +| `rules` | [`[EscalationRuleInput!]`](#escalationruleinput) | The steps of the escalation policy. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `escalationPolicy` | [`EscalationPolicyType`](#escalationpolicytype) | The escalation policy. | + ### `Mutation.exportRequirements` Input type: `ExportRequirementsInput` @@ -2588,6 +2612,31 @@ Input type: `IterationCadenceUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `iterationCadence` | [`IterationCadence`](#iterationcadence) | The updated iteration cadence. | +### `Mutation.iterationCreate` + +Input type: `iterationCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | The description of the iteration. | +| `dueDate` | [`String`](#string) | The end date of the iteration. | +| `groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | +| `iterationsCadenceId` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iterations cadence to be assigned to newly created iteration. | +| `projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | +| `startDate` | [`String`](#string) | The start date of the iteration. | +| `title` | [`String`](#string) | The title of the iteration. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `iteration` | [`Iteration`](#iteration) | The created iteration. | + ### `Mutation.iterationDelete` Input type: `IterationDeleteInput` @@ -2700,7 +2749,6 @@ Input type: `LabelCreateInput` | `description` | [`String`](#string) | Description of the label. | | `groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | | `projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | -| `removeOnClose` | [`Boolean`](#boolean) | Whether the label should be removed from an issue when the issue is closed. | | `title` | [`String!`](#string) | Title of the label. | #### Fields @@ -3414,30 +3462,6 @@ Input type: `ReleaseUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `release` | [`Release`](#release) | The release after mutation. | -### `Mutation.removeAwardEmoji` - -WARNING: -**Deprecated** in 13.2. -Use awardEmojiRemove. - -Input type: `RemoveAwardEmojiInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardableId` | [`AwardableID!`](#awardableid) | The global ID of the awardable resource. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `name` | [`String!`](#string) | The emoji name. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardEmoji` | [`AwardEmoji`](#awardemoji) | The award emoji after mutation. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | - ### `Mutation.removeProjectFromSecurityDashboard` Input type: `RemoveProjectFromSecurityDashboardInput` @@ -3478,54 +3502,75 @@ Input type: `RepositionImageDiffNoteInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `note` | [`Note`](#note) | The note after mutation. | -### `Mutation.revertVulnerabilityToDetected` +### `Mutation.runnerDelete` -WARNING: -**Deprecated** in 13.5. -Use vulnerabilityRevertToDetected. +Available only when feature flag `runner_graphql_query` is enabled. -Input type: `RevertVulnerabilityToDetectedInput` +Input type: `RunnerDeleteInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be reverted. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner to delete. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `vulnerability` | [`Vulnerability`](#vulnerability) | The vulnerability after revert. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -### `Mutation.runDastScan` +### `Mutation.runnerUpdate` -WARNING: -**Deprecated** in 13.4. -Use DastOnDemandScanCreate. +Available only when feature flag `runner_graphql_query` is enabled. -Input type: `RunDASTScanInput` +Input type: `RunnerUpdateInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `branch` | [`String!`](#string) | The branch to be associated with the scan. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `projectPath` | [`ID!`](#id) | The project the DAST scan belongs to. | -| `scanType` | [`DastScanTypeEnum!`](#dastscantypeenum) | The type of scan to be run. | -| `targetUrl` | [`String!`](#string) | The URL of the target to be scanned. | +| `accessLevel` | [`CiRunnerAccessLevel`](#cirunneraccesslevel) | Access level of the runner. | +| `active` | [`Boolean`](#boolean) | Indicates the runner is allowed to receive jobs. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of the runner. | +| `id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner to update. | +| `locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | +| `maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | +| `runUntagged` | [`Boolean`](#boolean) | Indicates the runner is able to run untagged jobs. | +| `tagList` | [`[String!]`](#string) | Tags associated with the runner. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `pipelineUrl` | [`String`](#string) | URL of the pipeline that was created. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `runner` | [`CiRunner`](#cirunner) | The runner after mutation. | + +### `Mutation.runnersRegistrationTokenReset` + +Available only when feature flag `runner_graphql_query` is enabled. + +Input type: `RunnersRegistrationTokenResetInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`ID`](#id) | ID of the project or group to reset the token for. Omit if resetting instance runner token. | +| `type` | [`CiRunnerType!`](#cirunnertype) | Scope of the object to reset the token for. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `token` | [`String`](#string) | The runner token after mutation. | ### `Mutation.terraformStateDelete` @@ -3656,7 +3701,6 @@ Input type: `TodoRestoreManyInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `todos` | [`[Todo!]!`](#todo) | Updated to-do items. | -| `updatedIds` **{warning-solid}** | [`[TodoID!]!`](#todoid) | **Deprecated:** Use to-do items. Deprecated in 13.2. | ### `Mutation.todosMarkAllDone` @@ -3675,32 +3719,6 @@ Input type: `TodosMarkAllDoneInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `todos` | [`[Todo!]!`](#todo) | Updated to-do items. | -| `updatedIds` **{warning-solid}** | [`[TodoID!]!`](#todoid) | **Deprecated:** Use to-do items. Deprecated in 13.2. | - -### `Mutation.toggleAwardEmoji` - -WARNING: -**Deprecated** in 13.2. -Use awardEmojiToggle. - -Input type: `ToggleAwardEmojiInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardableId` | [`AwardableID!`](#awardableid) | The global ID of the awardable resource. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `name` | [`String!`](#string) | The emoji name. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `awardEmoji` | [`AwardEmoji`](#awardemoji) | The award emoji after mutation. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `toggledOn` | [`Boolean!`](#boolean) | Indicates the status of the emoji. True if the toggle awarded the emoji, and false if the toggle removed the emoji. | ### `Mutation.updateAlertStatus` @@ -4251,6 +4269,29 @@ Some of the types in the schema exist solely to model connections. Each connecti has a distinct, named type, with a distinct named edge type. These are listed separately below. +#### `AgentConfigurationConnection` + +The connection type for [`AgentConfiguration`](#agentconfiguration). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[AgentConfigurationEdge]`](#agentconfigurationedge) | A list of edges. | +| `nodes` | [`[AgentConfiguration]`](#agentconfiguration) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `AgentConfigurationEdge` + +The edge type for [`AgentConfiguration`](#agentconfiguration). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`AgentConfiguration`](#agentconfiguration) | The item at the end of the edge. | + #### `AlertManagementAlertConnection` The connection type for [`AlertManagementAlert`](#alertmanagementalert). @@ -5037,28 +5078,51 @@ The edge type for [`DesignVersion`](#designversion). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`DesignVersion`](#designversion) | The item at the end of the edge. | -#### `DevopsAdoptionSegmentConnection` +#### `DevopsAdoptionEnabledNamespaceConnection` -The connection type for [`DevopsAdoptionSegment`](#devopsadoptionsegment). +The connection type for [`DevopsAdoptionEnabledNamespace`](#devopsadoptionenablednamespace). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `edges` | [`[DevopsAdoptionSegmentEdge]`](#devopsadoptionsegmentedge) | A list of edges. | -| `nodes` | [`[DevopsAdoptionSegment]`](#devopsadoptionsegment) | A list of nodes. | -| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| `edges` | [`[DevopsAdoptionEnabledNamespaceEdge]`](#devopsadoptionenablednamespaceedge) | A list of edges. | +| `nodes` | [`[DevopsAdoptionEnabledNamespace]`](#devopsadoptionenablednamespace) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -#### `DevopsAdoptionSegmentEdge` +#### `DevopsAdoptionEnabledNamespaceEdge` -The edge type for [`DevopsAdoptionSegment`](#devopsadoptionsegment). +The edge type for [`DevopsAdoptionEnabledNamespace`](#devopsadoptionenablednamespace). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `cursor` | [`String!`](#string) | A cursor for use in pagination. | -| `node` | [`DevopsAdoptionSegment`](#devopsadoptionsegment) | The item at the end of the edge. | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`DevopsAdoptionEnabledNamespace`](#devopsadoptionenablednamespace) | The item at the end of the edge. | + +#### `DevopsAdoptionSnapshotConnection` + +The connection type for [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[DevopsAdoptionSnapshotEdge]`](#devopsadoptionsnapshotedge) | A list of edges. | +| `nodes` | [`[DevopsAdoptionSnapshot]`](#devopsadoptionsnapshot) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `DevopsAdoptionSnapshotEdge` + +The edge type for [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot) | The item at the end of the edge. | #### `DiscussionConnection` @@ -5200,6 +5264,29 @@ The edge type for [`EpicList`](#epiclist). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`EpicList`](#epiclist) | The item at the end of the edge. | +#### `EscalationPolicyTypeConnection` + +The connection type for [`EscalationPolicyType`](#escalationpolicytype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[EscalationPolicyTypeEdge]`](#escalationpolicytypeedge) | A list of edges. | +| `nodes` | [`[EscalationPolicyType]`](#escalationpolicytype) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `EscalationPolicyTypeEdge` + +The edge type for [`EscalationPolicyType`](#escalationpolicytype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`EscalationPolicyType`](#escalationpolicytype) | The item at the end of the edge. | + #### `EventConnection` The connection type for [`Event`](#event). @@ -5711,6 +5798,29 @@ The edge type for [`Namespace`](#namespace). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`Namespace`](#namespace) | The item at the end of the edge. | +#### `NetworkPolicyConnection` + +The connection type for [`NetworkPolicy`](#networkpolicy). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[NetworkPolicyEdge]`](#networkpolicyedge) | A list of edges. | +| `nodes` | [`[NetworkPolicy]`](#networkpolicy) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `NetworkPolicyEdge` + +The edge type for [`NetworkPolicy`](#networkpolicy). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`NetworkPolicy`](#networkpolicy) | The item at the end of the edge. | + #### `NoteConnection` The connection type for [`Note`](#note). @@ -6265,6 +6375,29 @@ The edge type for [`Scan`](#scan). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`Scan`](#scan) | The item at the end of the edge. | +#### `ScanExecutionPolicyConnection` + +The connection type for [`ScanExecutionPolicy`](#scanexecutionpolicy). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[ScanExecutionPolicyEdge]`](#scanexecutionpolicyedge) | A list of edges. | +| `nodes` | [`[ScanExecutionPolicy]`](#scanexecutionpolicy) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `ScanExecutionPolicyEdge` + +The edge type for [`ScanExecutionPolicy`](#scanexecutionpolicy). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`ScanExecutionPolicy`](#scanexecutionpolicy) | The item at the end of the edge. | + #### `ScannedResourceConnection` The connection type for [`ScannedResource`](#scannedresource). @@ -6682,29 +6815,6 @@ The edge type for [`UserCore`](#usercore). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`UserCore`](#usercore) | The item at the end of the edge. | -#### `VulnerabilitiesCountByDayAndSeverityConnection` - -The connection type for [`VulnerabilitiesCountByDayAndSeverity`](#vulnerabilitiescountbydayandseverity). - -##### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `edges` | [`[VulnerabilitiesCountByDayAndSeverityEdge]`](#vulnerabilitiescountbydayandseverityedge) | A list of edges. | -| `nodes` | [`[VulnerabilitiesCountByDayAndSeverity]`](#vulnerabilitiescountbydayandseverity) | A list of nodes. | -| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | - -#### `VulnerabilitiesCountByDayAndSeverityEdge` - -The edge type for [`VulnerabilitiesCountByDayAndSeverity`](#vulnerabilitiescountbydayandseverity). - -##### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `cursor` | [`String!`](#string) | A cursor for use in pagination. | -| `node` | [`VulnerabilitiesCountByDayAndSeverity`](#vulnerabilitiescountbydayandseverity) | The item at the end of the edge. | - #### `VulnerabilitiesCountByDayConnection` The connection type for [`VulnerabilitiesCountByDay`](#vulnerabilitiescountbyday). @@ -6844,6 +6954,16 @@ Represents the access level of a relationship between a User and object that it | `integerValue` | [`Int`](#int) | Integer representation of access level. | | `stringValue` | [`AccessLevelEnum`](#accesslevelenum) | String representation of access level. | +### `AgentConfiguration` + +Configuration details for an Agent. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `agentName` | [`String`](#string) | Name of the agent. | + ### `AlertManagementAlert` Describes an alert from the project's Alert Management. @@ -7181,6 +7301,38 @@ Represents an epic on an issue board. #### Fields with arguments +##### `BoardEpic.ancestors` + +Ancestors (parents) of the epic. + +Returns [`EpicConnection`](#epicconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `authorUsername` | [`String`](#string) | Filter epics by author. | +| `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | +| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | +| `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | +| `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | +| `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | +| `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | +| `labelName` | [`[String!]`](#string) | Filter epics by labels. | +| `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | +| `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | +| `search` | [`String`](#string) | Search query for epic title or description. | +| `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | +| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | +| `state` | [`EpicState`](#epicstate) | Filter epics by state. | +| `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | + ##### `BoardEpic.children` Children (sub-epics) of the epic. @@ -7201,10 +7353,12 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | | `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | | `labelName` | [`[String!]`](#string) | Filter epics by labels. | | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | | `search` | [`String`](#string) | Search query for epic title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | | `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | @@ -7468,6 +7622,8 @@ Represents the total number of issues and their weights for a particular day. | `ipAddress` | [`String!`](#string) | IP address of the runner. | | `locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | | `maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | +| `privateProjectsMinutesCostFactor` | [`Float`](#float) | Private projects' "minutes cost factor" associated with the runner (GitLab.com only). | +| `publicProjectsMinutesCostFactor` | [`Float`](#float) | Public projects' "minutes cost factor" associated with the runner (GitLab.com only). | | `revision` | [`String!`](#string) | Revision of the runner. | | `runUntagged` | [`Boolean!`](#boolean) | Indicates the runner is able to run untagged jobs. | | `runnerType` | [`CiRunnerType!`](#cirunnertype) | Type of the runner. | @@ -7754,6 +7910,7 @@ Represents the current license. | ---- | ---- | ----------- | | `activatedAt` | [`Date`](#date) | Date when the license was activated. | | `billableUsersCount` | [`Int`](#int) | Number of billable users on the system. | +| `blockChangesAt` | [`Date`](#date) | Date, including grace period, when licensed features will be blocked. | | `company` | [`String`](#string) | Company of the licensee. | | `email` | [`String`](#string) | Email of the licensee. | | `expiresAt` | [`Date`](#date) | Date when the license expires. | @@ -7816,7 +7973,6 @@ Represents a DAST scanner profile. | Name | Type | Description | | ---- | ---- | ----------- | | `editPath` | [`String`](#string) | Relative web path to the edit page of a scanner profile. | -| `globalId` **{warning-solid}** | [`DastScannerProfileID!`](#dastscannerprofileid) | **Deprecated** in 13.6. Use `id`. | | `id` | [`DastScannerProfileID!`](#dastscannerprofileid) | ID of the DAST scanner profile. | | `profileName` | [`String`](#string) | Name of the DAST scanner profile. | | `referencedInSecurityPolicies` | [`[String!]`](#string) | List of security policy names that are referencing given project. | @@ -7834,15 +7990,15 @@ Represents a DAST Site Profile. | Name | Type | Description | | ---- | ---- | ----------- | -| `auth` | [`DastSiteProfileAuth`](#dastsiteprofileauth) | Target authentication details. Will always return `null` if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `auth` | [`DastSiteProfileAuth`](#dastsiteprofileauth) | Target authentication details. | | `editPath` | [`String`](#string) | Relative web path to the edit page of a site profile. | -| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. Will always return `null` if `security_dast_site_profiles_additional_fields` feature flag is disabled. | +| `excludedUrls` | [`[String!]`](#string) | The URLs to skip during an authenticated scan. | | `id` | [`DastSiteProfileID!`](#dastsiteprofileid) | ID of the site profile. | | `normalizedTargetUrl` | [`String`](#string) | Normalized URL of the target to be scanned. | | `profileName` | [`String`](#string) | The name of the site profile. | | `referencedInSecurityPolicies` | [`[String!]`](#string) | List of security policy names that are referencing given project. | -| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. Will always return `null` if `security_dast_site_profiles_additional_fields` feature flag is disabled. | -| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. Will always return `null` if `security_dast_site_profiles_api_option` feature flag is disabled. | +| `requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | +| `targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | The type of target to be scanned. | | `targetUrl` | [`String`](#string) | The URL of the target to be scanned. | | `userPermissions` | [`DastSiteProfilePermissions!`](#dastsiteprofilepermissions) | Permissions for the current user on the resource. | | `validationStatus` | [`DastSiteProfileValidationStatusEnum`](#dastsiteprofilevalidationstatusenum) | The current validation status of the site profile. | @@ -8151,17 +8307,37 @@ four standard [pagination arguments](#connection-pagination-arguments): | `text` | [`String`](#string) | Text of the status. | | `tooltip` | [`String`](#string) | Tooltip associated with the status. | -### `DevopsAdoptionSegment` +### `DevopsAdoptionEnabledNamespace` -Segment. +Enabled namespace for DevopsAdoption. #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `id` | [`ID!`](#id) | ID of the segment. | -| `latestSnapshot` | [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot) | The latest adoption metrics for the segment. | -| `namespace` | [`Namespace`](#namespace) | Segment namespace. | +| `displayNamespace` | [`Namespace`](#namespace) | Namespace where data should be displayed. | +| `id` | [`ID!`](#id) | ID of the enabled namespace. | +| `latestSnapshot` | [`DevopsAdoptionSnapshot`](#devopsadoptionsnapshot) | Metrics snapshot for previous month for the enabled namespace. | +| `namespace` | [`Namespace`](#namespace) | Namespace which should be calculated. | + +#### Fields with arguments + +##### `DevopsAdoptionEnabledNamespace.snapshots` + +Data snapshots of the namespace. + +Returns [`DevopsAdoptionSnapshotConnection`](#devopsadoptionsnapshotconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `endTimeAfter` | [`Time`](#time) | Filter to snapshots with month end after the provided date. | +| `endTimeBefore` | [`Time`](#time) | Filter to snapshots with month end before the provided date. | ### `DevopsAdoptionSnapshot` @@ -8336,6 +8512,38 @@ Represents an epic. #### Fields with arguments +##### `Epic.ancestors` + +Ancestors (parents) of the epic. + +Returns [`EpicConnection`](#epicconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `authorUsername` | [`String`](#string) | Filter epics by author. | +| `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | +| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | +| `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | +| `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | +| `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | +| `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | +| `labelName` | [`[String!]`](#string) | Filter epics by labels. | +| `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | +| `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | +| `search` | [`String`](#string) | Search query for epic title or description. | +| `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | +| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | +| `state` | [`EpicState`](#epicstate) | Filter epics by state. | +| `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | + ##### `Epic.children` Children (sub-epics) of the epic. @@ -8356,10 +8564,12 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | | `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | | `labelName` | [`[String!]`](#string) | Filter epics by labels. | | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | | `search` | [`String`](#string) | Search query for epic title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | | `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | @@ -8426,6 +8636,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `epicFilters` | [`EpicFilters`](#epicfilters) | Filters applied when getting epic metadata in the epic board list. | | `id` | [`BoardsEpicListID`](#boardsepiclistid) | Find an epic board list by ID. | ### `EpicDescendantCount` @@ -8610,6 +8821,32 @@ Check permissions for the current user on an epic. | `readEpicIid` | [`Boolean!`](#boolean) | Indicates the user can perform `read_epic_iid` on this resource. | | `updateEpic` | [`Boolean!`](#boolean) | Indicates the user can perform `update_epic` on this resource. | +### `EscalationPolicyType` + +Represents an escalation policy. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `description` | [`String`](#string) | The description of the escalation policy. | +| `id` | [`IncidentManagementEscalationPolicyID`](#incidentmanagementescalationpolicyid) | ID of the escalation policy. | +| `name` | [`String`](#string) | The name of the escalation policy. | +| `rules` | [`[EscalationRuleType!]`](#escalationruletype) | Steps of the escalation policy. | + +### `EscalationRuleType` + +Represents an escalation rule for an escalation policy. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `elapsedTimeSeconds` | [`Int`](#int) | The time in seconds before the rule is activated. | +| `id` | [`IncidentManagementEscalationRuleID`](#incidentmanagementescalationruleid) | ID of the escalation policy. | +| `oncallSchedule` | [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule) | The on-call schedule to notify. | +| `status` | [`EscalationRuleStatus`](#escalationrulestatus) | The status required to prevent the rule from activating. | + ### `Event` Representing an event. @@ -8930,10 +9167,12 @@ Returns [`Epic`](#epic). | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | | `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | | `labelName` | [`[String!]`](#string) | Filter epics by labels. | | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | | `search` | [`String`](#string) | Search query for epic title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | | `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | @@ -8972,10 +9211,12 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., [1, 2]. | +| `includeAncestorGroups` | [`Boolean`](#boolean) | Include epics from ancestor groups. | | `includeDescendantGroups` | [`Boolean`](#boolean) | Include epics from descendant groups. | | `labelName` | [`[String!]`](#string) | Filter epics by labels. | | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +| `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | | `search` | [`String`](#string) | Search query for epic title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | | `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | @@ -9268,27 +9509,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endDate` | [`ISO8601Date!`](#iso8601date) | Last day for which to fetch vulnerability history. | | `startDate` | [`ISO8601Date!`](#iso8601date) | First day for which to fetch vulnerability history. | -##### `Group.vulnerabilitiesCountByDayAndSeverity` - -Number of vulnerabilities per severity level, per day, for the projects in the group and its subgroups. - -WARNING: -**Deprecated** in 13.3. -Use `vulnerabilitiesCountByDay`. - -Returns [`VulnerabilitiesCountByDayAndSeverityConnection`](#vulnerabilitiescountbydayandseverityconnection). - -This field returns a [connection](#connections). It accepts the -four standard [pagination arguments](#connection-pagination-arguments): -`before: String`, `after: String`, `first: Int`, `last: Int`. - -###### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `endDate` | [`ISO8601Date!`](#iso8601date) | Last day for which to fetch vulnerability history. | -| `startDate` | [`ISO8601Date!`](#iso8601date) | First day for which to fetch vulnerability history. | - ##### `Group.vulnerabilityGrades` Represents vulnerable project counts for each grade. @@ -9311,9 +9531,12 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | Name | Type | Description | | ---- | ---- | ----------- | +| `hasIssues` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have issues. | +| `hasResolution` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have a resolution. | | `projectId` | [`[ID!]`](#id) | Filter vulnerabilities by project. | | `reportType` | [`[VulnerabilityReportType!]`](#vulnerabilityreporttype) | Filter vulnerabilities by report type. | | `scanner` | [`[String!]`](#string) | Filter vulnerabilities by scanner. | +| `scannerId` | [`[VulnerabilitiesScannerID!]`](#vulnerabilitiesscannerid) | Filter vulnerabilities by scanner ID. | | `severity` | [`[VulnerabilitySeverity!]`](#vulnerabilityseverity) | Filter vulnerabilities by severity. | | `state` | [`[VulnerabilityState!]`](#vulnerabilitystate) | Filter vulnerabilities by state. | @@ -9332,7 +9555,7 @@ Represents a Group Membership. | `group` | [`Group`](#group) | Group that a User is a member of. | | `id` | [`ID!`](#id) | ID of the member. | | `updatedAt` | [`Time`](#time) | Date and time the membership was last updated. | -| `user` | [`UserCore!`](#usercore) | User that is associated with the member object. | +| `user` | [`UserCore`](#usercore) | User that is associated with the member object. | | `userPermissions` | [`GroupPermissions!`](#grouppermissions) | Permissions for the current user on the resource. | ### `GroupPermissions` @@ -9463,12 +9686,27 @@ A block of time for which a participant is on-call. | Name | Type | Description | | ---- | ---- | ----------- | -| `projects` | [`ProjectConnection!`](#projectconnection) | Projects selected in Instance Security Dashboard. (see [Connections](#connections)) | | `vulnerabilityGrades` | [`[VulnerableProjectsByGrade!]!`](#vulnerableprojectsbygrade) | Represents vulnerable project counts for each grade. | | `vulnerabilityScanners` | [`VulnerabilityScannerConnection`](#vulnerabilityscannerconnection) | Vulnerability scanners reported on the vulnerabilities from projects selected in Instance Security Dashboard. (see [Connections](#connections)) | #### Fields with arguments +##### `InstanceSecurityDashboard.projects` + +Projects selected in Instance Security Dashboard. + +Returns [`ProjectConnection!`](#projectconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `search` | [`String`](#string) | Search query for project name, path, or description. | + ##### `InstanceSecurityDashboard.vulnerabilitySeveritiesCount` Counts for each vulnerability severity from projects selected in Instance Security Dashboard. @@ -9479,9 +9717,12 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | Name | Type | Description | | ---- | ---- | ----------- | +| `hasIssues` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have issues. | +| `hasResolution` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have a resolution. | | `projectId` | [`[ID!]`](#id) | Filter vulnerabilities by project. | | `reportType` | [`[VulnerabilityReportType!]`](#vulnerabilityreporttype) | Filter vulnerabilities by report type. | | `scanner` | [`[String!]`](#string) | Filter vulnerabilities by scanner. | +| `scannerId` | [`[VulnerabilitiesScannerID!]`](#vulnerabilitiesscannerid) | Filter vulnerabilities by scanner ID. | | `severity` | [`[VulnerabilitySeverity!]`](#vulnerabilityseverity) | Filter vulnerabilities by severity. | | `state` | [`[VulnerabilityState!]`](#vulnerabilitystate) | Filter vulnerabilities by state. | @@ -9743,7 +9984,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `description` | [`String`](#string) | Description of the label (Markdown rendered as HTML for caching). | | `descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | | `id` | [`ID!`](#id) | Label ID. | -| `removeOnClose` | [`Boolean!`](#boolean) | Whether the label should be removed from an issue when the issue is closed. | | `textColor` | [`String!`](#string) | Text color of the label. | | `title` | [`String!`](#string) | Content of the label. | | `updatedAt` | [`Time!`](#time) | When this label was last updated. | @@ -9774,6 +10014,7 @@ Represents an entry from the Cloud License history. | Name | Type | Description | | ---- | ---- | ----------- | | `activatedAt` | [`Date`](#date) | Date when the license was activated. | +| `blockChangesAt` | [`Date`](#date) | Date, including grace period, when licensed features will be blocked. | | `company` | [`String`](#string) | Company of the licensee. | | `email` | [`String`](#string) | Email of the licensee. | | `expiresAt` | [`Date`](#date) | Date when the license expires. | @@ -9837,6 +10078,8 @@ Maven metadata. | `hasCi` | [`Boolean!`](#boolean) | Indicates if the merge request has CI. | | `hasSecurityReports` | [`Boolean!`](#boolean) | Indicates if the source branch has any security reports. | | `headPipeline` | [`Pipeline`](#pipeline) | The pipeline running on the branch HEAD of the merge request. | +| `humanTimeEstimate` | [`String`](#string) | Human-readable time estimate of the merge request. | +| `humanTotalTimeSpent` | [`String`](#string) | Human-readable total time reported as spent on the merge request. | | `id` | [`ID!`](#id) | ID of the merge request. | | `iid` | [`String!`](#string) | Internal ID of the merge request. | | `inProgressMergeCommitSha` | [`String`](#string) | Commit SHA of the merge request if merge is in progress. | @@ -9844,7 +10087,8 @@ Maven metadata. | `mergeCommitSha` | [`String`](#string) | SHA of the merge request commit (set once merged). | | `mergeError` | [`String`](#string) | Error message due to a merge error. | | `mergeOngoing` | [`Boolean!`](#boolean) | Indicates if a merge is currently occurring. | -| `mergeStatus` | [`String`](#string) | Status of the merge request. | +| `mergeStatus` **{warning-solid}** | [`String`](#string) | **Deprecated** in 14.0. This was renamed. Use: [`MergeRequest.mergeStatusEnum`](#mergerequestmergestatusenum). | +| `mergeStatusEnum` | [`MergeStatus`](#mergestatus) | Merge status of the merge request. | | `mergeTrainsCount` | [`Int`](#int) | Number of merge requests in the merge train. | | `mergeUser` | [`UserCore`](#usercore) | User who merged this merge request. | | `mergeWhenPipelineSucceeds` | [`Boolean`](#boolean) | Indicates if the merge has been set to be merged when its pipeline succeeds (MWPS). | @@ -10493,6 +10737,21 @@ four standard [pagination arguments](#connection-pagination-arguments): | `search` | [`String`](#string) | Search project with most similar names or paths. | | `sort` | [`NamespaceProjectSort`](#namespaceprojectsort) | Sort projects by this criteria. | +### `NetworkPolicy` + +Represents the network policy. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `enabled` | [`Boolean!`](#boolean) | Indicates whether this policy is enabled. | +| `fromAutoDevops` | [`Boolean!`](#boolean) | Indicates whether this policy is created from AutoDevops. | +| `name` | [`String!`](#string) | Name of the policy. | +| `namespace` | [`String!`](#string) | Namespace of the policy. | +| `updatedAt` | [`Time!`](#time) | Timestamp of when the policy YAML was last updated. | +| `yaml` | [`String!`](#string) | YAML definition of the policy. | + ### `Note` #### Fields @@ -10877,6 +11136,7 @@ Represents vulnerability finding of a security report on the pipeline. | `scanner` | [`VulnerabilityScanner`](#vulnerabilityscanner) | Scanner metadata for the vulnerability. | | `severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the vulnerability finding. | | `solution` | [`String`](#string) | URL to the vulnerability's details page. | +| `state` | [`VulnerabilityState`](#vulnerabilitystate) | The finding status. | | `uuid` | [`String`](#string) | Name of the vulnerability finding. | ### `Project` @@ -10886,6 +11146,7 @@ Represents vulnerability finding of a security report on the pipeline. | Name | Type | Description | | ---- | ---- | ----------- | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for the repository in bytes. | +| `agentConfigurations` | [`AgentConfigurationConnection`](#agentconfigurationconnection) | Agent configurations defined by the project. (see [Connections](#connections)) | | `allowMergeOnSkippedPipeline` | [`Boolean`](#boolean) | If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs. | | `apiFuzzingCiConfiguration` | [`ApiFuzzingCiConfiguration`](#apifuzzingciconfiguration) | API fuzzing configuration for the project. | | `archived` | [`Boolean`](#boolean) | Indicates the archived status of the project. | @@ -10911,6 +11172,7 @@ Represents vulnerability finding of a security report on the pipeline. | `httpUrlToRepo` | [`String`](#string) | URL to connect to the project via HTTPS. | | `id` | [`ID!`](#id) | ID of the project. | | `importStatus` | [`String`](#string) | Status of import background job of the project. | +| `incidentManagementEscalationPolicies` | [`EscalationPolicyTypeConnection`](#escalationpolicytypeconnection) | Incident Management escalation policies of the project. (see [Connections](#connections)) | | `issuesEnabled` | [`Boolean`](#boolean) | Indicates if Issues are enabled for the current user. | | `jiraImportStatus` | [`String`](#string) | Status of Jira import background job of the project. | | `jiraImports` | [`JiraImportConnection`](#jiraimportconnection) | Jira imports into the project. (see [Connections](#connections)) | @@ -10937,6 +11199,7 @@ Represents vulnerability finding of a security report on the pipeline. | `requestAccessEnabled` | [`Boolean`](#boolean) | Indicates if users can request member access to the project. | | `requirementStatesCount` | [`RequirementStatesCount`](#requirementstatescount) | Number of requirements for the project by their state. | | `sastCiConfiguration` | [`SastCiConfiguration`](#sastciconfiguration) | SAST CI configuration for the project. | +| `scanExecutionPolicies` | [`ScanExecutionPolicyConnection`](#scanexecutionpolicyconnection) | Scan Execution Policies of the project. (see [Connections](#connections)) | | `securityDashboardPath` | [`String`](#string) | Path to project's security dashboard. | | `securityScanners` | [`SecurityScanners`](#securityscanners) | Information about security analyzers used in the project. | | `sentryErrors` | [`SentryErrorCollection`](#sentryerrorcollection) | Paginated collection of Sentry errors on the project. | @@ -11184,6 +11447,18 @@ four standard [pagination arguments](#connection-pagination-arguments): | `search` | [`String`](#string) | Search query for environment name. | | `states` | [`[String!]`](#string) | States of environments that should be included in result. | +##### `Project.incidentManagementEscalationPolicy` + +Incident Management escalation policy of the project. + +Returns [`EscalationPolicyType`](#escalationpolicytype). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`IncidentManagementEscalationPolicyID!`](#incidentmanagementescalationpolicyid) | ID of the escalation policy. | + ##### `Project.incidentManagementOncallSchedules` Incident Management On-call schedules of the project. @@ -11454,6 +11729,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `title` | [`String`](#string) | The title of the milestone. | +##### `Project.networkPolicies` + +Network Policies of the project. + +Returns [`NetworkPolicyConnection`](#networkpolicyconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `environmentId` | [`EnvironmentID`](#environmentid) | The global ID of the environment to filter policies. | + ##### `Project.packages` Packages of the project. @@ -11616,8 +11907,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `active` | [`Boolean`](#boolean) | Indicates if the service is active. | -| `type` | [`ServiceType`](#servicetype) | Class name of the service. | +| `active` | [`Boolean`](#boolean) | Indicates if the integration is active. | +| `type` | [`ServiceType`](#servicetype) | Type of integration. | ##### `Project.snippets` @@ -11699,9 +11990,12 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | Name | Type | Description | | ---- | ---- | ----------- | +| `hasIssues` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have issues. | +| `hasResolution` | [`Boolean`](#boolean) | Filter vulnerabilities that do or do not have a resolution. | | `projectId` | [`[ID!]`](#id) | Filter vulnerabilities by project. | | `reportType` | [`[VulnerabilityReportType!]`](#vulnerabilityreporttype) | Filter vulnerabilities by report type. | | `scanner` | [`[String!]`](#string) | Filter vulnerabilities by scanner. | +| `scannerId` | [`[VulnerabilitiesScannerID!]`](#vulnerabilitiesscannerid) | Filter vulnerabilities by scanner ID. | | `severity` | [`[VulnerabilitySeverity!]`](#vulnerabilityseverity) | Filter vulnerabilities by severity. | | `state` | [`[VulnerabilityState!]`](#vulnerabilitystate) | Filter vulnerabilities by state. | @@ -11711,6 +12005,7 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | Name | Type | Description | | ---- | ---- | ----------- | +| `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI job tokens generated in this project have restricted access to resources. | | `keepLatestArtifact` | [`Boolean`](#boolean) | Whether to keep the latest builds artifacts. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Whether merge pipelines are enabled. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Whether merge trains are enabled. | @@ -11731,7 +12026,7 @@ Represents a Project Membership. | `id` | [`ID!`](#id) | ID of the member. | | `project` | [`Project`](#project) | Project that User is a member of. | | `updatedAt` | [`Time`](#time) | Date and time the membership was last updated. | -| `user` | [`UserCore!`](#usercore) | User that is associated with the member object. | +| `user` | [`UserCore`](#usercore) | User that is associated with the member object. | | `userPermissions` | [`ProjectPermissions!`](#projectpermissions) | Permissions for the current user on the resource. | ### `ProjectPermissions` @@ -11821,6 +12116,17 @@ Represents rules that commit pushes must follow. | ---- | ---- | ----------- | | `rejectUnsignedCommits` | [`Boolean!`](#boolean) | Indicates whether commits not signed through GPG will be rejected. | +### `PypiMetadata` + +Pypi metadata. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`PackagesPypiMetadatumID!`](#packagespypimetadatumid) | ID of the metadatum. | +| `requiredPython` | [`String`](#string) | Required Python version of the Pypi package. | + ### `RecentFailures` Recent failure history of a test case. @@ -12091,18 +12397,6 @@ Counts of requirements by their state. | `uploadsSize` | [`Float!`](#float) | The uploads size in bytes. | | `wikiSize` | [`Float!`](#float) | The wiki size in bytes. | -### `RunDASTScanPayload` - -Autogenerated return type of RunDASTScan. - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `pipelineUrl` | [`String`](#string) | URL of the pipeline that was created. | - ### `RunnerArchitecture` #### Fields @@ -12196,6 +12490,20 @@ Represents the security scan information. | `errors` | [`[String!]!`](#string) | List of errors. | | `name` | [`String!`](#string) | Name of the scan. | +### `ScanExecutionPolicy` + +Represents the scan execution policy. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `description` | [`String!`](#string) | Description of the policy. | +| `enabled` | [`Boolean!`](#boolean) | Indicates whether this policy is enabled. | +| `name` | [`String!`](#string) | Name of the policy. | +| `updatedAt` | [`Time!`](#time) | Timestamp of when the policy YAML was last updated. | +| `yaml` | [`String!`](#string) | YAML definition of the policy. | + ### `ScannedResource` Represents a resource scanned by a security scan. @@ -12431,7 +12739,6 @@ Represents a snippet entry. | Name | Type | Description | | ---- | ---- | ----------- | | `author` | [`UserCore`](#usercore) | The owner of the snippet. | -| `blob` **{warning-solid}** | [`SnippetBlob!`](#snippetblob) | **Deprecated** in 13.3. Use `blobs`. | | `createdAt` | [`Time!`](#time) | Timestamp this snippet was created. | | `description` | [`String`](#string) | Description of the snippet. | | `descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | @@ -13049,18 +13356,6 @@ Represents the count of vulnerabilities by severity on a particular day. This da | `total` | [`Int!`](#int) | Total number of vulnerabilities on a particular day. | | `unknown` | [`Int!`](#int) | Total number of vulnerabilities on a particular day with unknown severity. | -### `VulnerabilitiesCountByDayAndSeverity` - -Represents the number of vulnerabilities for a particular severity on a particular day. This data is retained for 365 days. - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `count` | [`Int`](#int) | Number of vulnerabilities. | -| `day` | [`ISO8601Date`](#iso8601date) | Date for the count. | -| `severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the counted vulnerabilities. | - ### `Vulnerability` Represents a vulnerability. @@ -13537,10 +13832,10 @@ Values for sorting alerts. | `UPDATED_DESC` | Updated at descending order. | | `UPDATED_TIME_ASC` | Created time by ascending order. | | `UPDATED_TIME_DESC` | Created time by descending order. | -| `created_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_ASC`. Deprecated in 13.5. | -| `created_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_DESC`. Deprecated in 13.5. | -| `updated_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_ASC`. Deprecated in 13.5. | -| `updated_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_DESC`. Deprecated in 13.5. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | ### `AlertManagementDomainFilter` @@ -13699,7 +13994,9 @@ Values for sorting runners. | Value | Description | | ----- | ----------- | | `CONTACTED_ASC` | Ordered by contacted_at in ascending order. | -| `CREATED_DESC` | Ordered by created_date in descending order. | +| `CONTACTED_DESC` | Ordered by contacted_at in descending order. | +| `CREATED_ASC` | Ordered by created_at in ascending order. | +| `CREATED_DESC` | Ordered by created_at in descending order. | ### `CiRunnerStatus` @@ -13810,10 +14107,10 @@ Values for sorting container repositories. | `NAME_DESC` | Name by descending order. | | `UPDATED_ASC` | Updated at ascending order. | | `UPDATED_DESC` | Updated at descending order. | -| `created_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_ASC`. Deprecated in 13.5. | -| `created_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_DESC`. Deprecated in 13.5. | -| `updated_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_ASC`. Deprecated in 13.5. | -| `updated_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_DESC`. Deprecated in 13.5. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | ### `ContainerRepositoryStatus` @@ -13935,10 +14232,10 @@ Roadmap sort values. | `END_DATE_DESC` | Sort by end date in descending order. | | `START_DATE_ASC` | Sort by start date in ascending order. | | `START_DATE_DESC` | Sort by start date in descending order. | -| `end_date_asc` **{warning-solid}** | **Deprecated:** Use END_DATE_ASC. Deprecated in 13.11. | -| `end_date_desc` **{warning-solid}** | **Deprecated:** Use END_DATE_DESC. Deprecated in 13.11. | -| `start_date_asc` **{warning-solid}** | **Deprecated:** Use START_DATE_ASC. Deprecated in 13.11. | -| `start_date_desc` **{warning-solid}** | **Deprecated:** Use START_DATE_DESC. Deprecated in 13.11. | +| `end_date_asc` **{warning-solid}** | **Deprecated** in 13.11. Use END_DATE_ASC. | +| `end_date_desc` **{warning-solid}** | **Deprecated** in 13.11. Use END_DATE_DESC. | +| `start_date_asc` **{warning-solid}** | **Deprecated** in 13.11. Use START_DATE_ASC. | +| `start_date_desc` **{warning-solid}** | **Deprecated** in 13.11. Use START_DATE_DESC. | ### `EpicState` @@ -13968,6 +14265,15 @@ Epic ID wildcard values. | `ANY` | Any epic is assigned. | | `NONE` | No epic is assigned. | +### `EscalationRuleStatus` + +Escalation rule statuses. + +| Value | Description | +| ----- | ----------- | +| `ACKNOWLEDGED` | . | +| `RESOLVED` | . | + ### `EventAction` Event action. @@ -14058,10 +14364,10 @@ Values for sorting issues. | `UPDATED_DESC` | Updated at descending order. | | `WEIGHT_ASC` | Weight by ascending order. | | `WEIGHT_DESC` | Weight by descending order. | -| `created_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_ASC`. Deprecated in 13.5. | -| `created_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_DESC`. Deprecated in 13.5. | -| `updated_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_ASC`. Deprecated in 13.5. | -| `updated_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_DESC`. Deprecated in 13.5. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | ### `IssueState` @@ -14133,7 +14439,6 @@ Iteration ID wildcard values. | `DEPENDENCY_SCANNING` | DEPENDENCY SCANNING job artifact file type. | | `DOTENV` | DOTENV job artifact file type. | | `JUNIT` | JUNIT job artifact file type. | -| `LICENSE_MANAGEMENT` | LICENSE MANAGEMENT job artifact file type. | | `LICENSE_SCANNING` | LICENSE SCANNING job artifact file type. | | `LOAD_PERFORMANCE` | LOAD PERFORMANCE job artifact file type. | | `LSIF` | LSIF job artifact file type. | @@ -14211,10 +14516,10 @@ Values for sorting merge requests. | `PRIORITY_DESC` | Priority by descending order. | | `UPDATED_ASC` | Updated at ascending order. | | `UPDATED_DESC` | Updated at descending order. | -| `created_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_ASC`. Deprecated in 13.5. | -| `created_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_DESC`. Deprecated in 13.5. | -| `updated_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_ASC`. Deprecated in 13.5. | -| `updated_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_DESC`. Deprecated in 13.5. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | ### `MergeRequestState` @@ -14228,6 +14533,18 @@ State of a GitLab merge request. | `merged` | Merge request has been merged. | | `opened` | In open state. | +### `MergeStatus` + +Representation of whether a GitLab merge request can be merged. + +| Value | Description | +| ----- | ----------- | +| `CANNOT_BE_MERGED` | There are conflicts between the source and target branches. | +| `CANNOT_BE_MERGED_RECHECK` | Currently unchecked. The previous state was `CANNOT_BE_MERGED`. | +| `CAN_BE_MERGED` | There are no conflicts between the source and target branches. | +| `CHECKING` | Currently checking for mergeability. | +| `UNCHECKED` | Merge status has not been checked. | + ### `MergeStrategyEnum` | Value | Description | @@ -14301,6 +14618,8 @@ Values for sorting group packages. | `CREATED_DESC` | Ordered by created_at in descending order. | | `NAME_ASC` | Ordered by name in ascending order. | | `NAME_DESC` | Ordered by name in descending order. | +| `PROJECT_PATH_ASC` | Ordered by project path in ascending order. | +| `PROJECT_PATH_DESC` | Ordered by project path in descending order. | | `TYPE_ASC` | Ordered by type in ascending order. | | `TYPE_DESC` | Ordered by type in descending order. | | `VERSION_ASC` | Ordered by version in ascending order. | @@ -14533,10 +14852,10 @@ Type of a snippet blob input action. | Value | Description | | ----- | ----------- | -| `create` | | -| `delete` | | -| `move` | | -| `update` | | +| `create` | Create a snippet blob. | +| `delete` | Delete a snippet blob. | +| `move` | Move a snippet blob. | +| `update` | Update a snippet blob. | ### `Sort` @@ -14548,10 +14867,10 @@ Common sort values. | `CREATED_DESC` | Created at descending order. | | `UPDATED_ASC` | Updated at ascending order. | | `UPDATED_DESC` | Updated at descending order. | -| `created_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_ASC`. Deprecated in 13.5. | -| `created_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `CREATED_DESC`. Deprecated in 13.5. | -| `updated_asc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_ASC`. Deprecated in 13.5. | -| `updated_desc` **{warning-solid}** | **Deprecated:** This was renamed. Please use `UPDATED_DESC`. Deprecated in 13.5. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | ### `TestCaseStatus` @@ -14618,7 +14937,6 @@ Name of the feature that the callout is for. | ----- | ----------- | | `ACCOUNT_RECOVERY_REGULAR_CHECK` | Callout feature name for account_recovery_regular_check. | | `ACTIVE_USER_COUNT_THRESHOLD` | Callout feature name for active_user_count_threshold. | -| `ADMIN_INTEGRATIONS_MOVED` | Callout feature name for admin_integrations_moved. | | `BUY_PIPELINE_MINUTES_NOTIFICATION_DOT` | Callout feature name for buy_pipeline_minutes_notification_dot. | | `CANARY_DEPLOYMENT` | Callout feature name for canary_deployment. | | `CLUSTER_SECURITY_WARNING` | Callout feature name for cluster_security_warning. | @@ -14635,6 +14953,7 @@ Name of the feature that the callout is for. | `PIPELINE_NEEDS_BANNER` | Callout feature name for pipeline_needs_banner. | | `PIPELINE_NEEDS_HOVER_TIP` | Callout feature name for pipeline_needs_hover_tip. | | `REGISTRATION_ENABLED_CALLOUT` | Callout feature name for registration_enabled_callout. | +| `SECURITY_CONFIGURATION_UPGRADE_BANNER` | Callout feature name for security_configuration_upgrade_banner. | | `SERVICE_TEMPLATES_DEPRECATED_CALLOUT` | Callout feature name for service_templates_deprecated_callout. | | `SUGGEST_PIPELINE` | Callout feature name for suggest_pipeline. | | `SUGGEST_POPOVER_DISMISSED` | Callout feature name for suggest_popover_dismissed. | @@ -14642,7 +14961,6 @@ Name of the feature that the callout is for. | `THREAT_MONITORING_INFO` | Callout feature name for threat_monitoring_info. | | `ULTIMATE_TRIAL` | Callout feature name for ultimate_trial. | | `UNFINISHED_TAG_CLEANUP_CALLOUT` | Callout feature name for unfinished_tag_cleanup_callout. | -| `WEBHOOKS_MOVED` | Callout feature name for webhooks_moved. | | `WEB_IDE_ALERT_DISMISSED` | Callout feature name for web_ide_alert_dismissed. | | `WEB_IDE_CI_ENVIRONMENTS_GUIDANCE` | Callout feature name for web_ide_ci_environments_guidance. | @@ -14668,9 +14986,9 @@ Possible states of a user. | Value | Description | | ----- | ----------- | -| `internal` | | -| `private` | | -| `public` | | +| `internal` | The snippet is visible for any logged in user except external users. | +| `private` | The snippet is visible only to the snippet creator. | +| `public` | The snippet can be accessed without any authentication. | ### `VulnerabilityDismissalReason` @@ -14802,11 +15120,11 @@ A `AlertManagementHttpIntegrationID` is a global ID. It is encoded as a string. An example `AlertManagementHttpIntegrationID` is: `"gid://gitlab/AlertManagement::HttpIntegration/1"`. -### `AnalyticsDevopsAdoptionSegmentID` +### `AnalyticsDevopsAdoptionEnabledNamespaceID` -A `AnalyticsDevopsAdoptionSegmentID` is a global ID. It is encoded as a string. +A `AnalyticsDevopsAdoptionEnabledNamespaceID` is a global ID. It is encoded as a string. -An example `AnalyticsDevopsAdoptionSegmentID` is: `"gid://gitlab/Analytics::DevopsAdoption::Segment/1"`. +An example `AnalyticsDevopsAdoptionEnabledNamespaceID` is: `"gid://gitlab/Analytics::DevopsAdoption::EnabledNamespace/1"`. ### `AwardableID` @@ -15015,6 +15333,18 @@ Represents a unique identifier that is Base64 obfuscated. It is often used to re An ISO 8601-encoded date. +### `IncidentManagementEscalationPolicyID` + +A `IncidentManagementEscalationPolicyID` is a global ID. It is encoded as a string. + +An example `IncidentManagementEscalationPolicyID` is: `"gid://gitlab/IncidentManagement::EscalationPolicy/1"`. + +### `IncidentManagementEscalationRuleID` + +A `IncidentManagementEscalationRuleID` is a global ID. It is encoded as a string. + +An example `IncidentManagementEscalationRuleID` is: `"gid://gitlab/IncidentManagement::EscalationRule/1"`. + ### `IncidentManagementOncallParticipantID` A `IncidentManagementOncallParticipantID` is a global ID. It is encoded as a string. @@ -15048,6 +15378,7 @@ An example `IssueID` is: `"gid://gitlab/Issue/1"`. A `IterationID` is a global ID. It is encoded as a string. An example `IterationID` is: `"gid://gitlab/Iteration/1"`. +The older format `"gid://gitlab/EEIteration/1"` was deprecated in 13.3. ### `IterationsCadenceID` @@ -15153,6 +15484,12 @@ A `PackagesPackageID` is a global ID. It is encoded as a string. An example `PackagesPackageID` is: `"gid://gitlab/Packages::Package/1"`. +### `PackagesPypiMetadatumID` + +A `PackagesPypiMetadatumID` is a global ID. It is encoded as a string. + +An example `PackagesPypiMetadatumID` is: `"gid://gitlab/Packages::Pypi::Metadatum/1"`. + ### `PathLockID` A `PathLockID` is a global ID. It is encoded as a string. @@ -15284,6 +15621,7 @@ One of: - [`ConanMetadata`](#conanmetadata) - [`MavenMetadata`](#mavenmetadata) - [`NugetMetadata`](#nugetmetadata) +- [`PypiMetadata`](#pypimetadata) #### `VulnerabilityDetail` @@ -15439,7 +15777,7 @@ Implementations: | `expiresAt` | [`Time`](#time) | Date and time the membership expires. | | `id` | [`ID!`](#id) | ID of the member. | | `updatedAt` | [`Time`](#time) | Date and time the membership was last updated. | -| `user` | [`UserCore!`](#usercore) | User that is associated with the member object. | +| `user` | [`UserCore`](#usercore) | User that is associated with the member object. | #### `Noteable` @@ -15843,6 +16181,18 @@ A node of an epic tree. | `newParentId` | [`EpicID`](#epicid) | ID of the new parent epic. | | `relativePosition` | [`MoveType`](#movetype) | The type of the switch, after or before allowed. | +### `EscalationRuleInput` + +Represents an escalation rule. + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `elapsedTimeSeconds` | [`Int!`](#int) | The time in seconds before the rule is activated. | +| `oncallScheduleIid` | [`ID!`](#id) | The on-call schedule to notify. | +| `status` | [`EscalationRuleStatus!`](#escalationrulestatus) | The status required to prevent the rule from activating. | + ### `JiraUsersMappingInputType` #### Arguments @@ -15890,6 +16240,16 @@ A node of an epic tree. | `labelName` | [`[String]`](#string) | Filter by label name. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | +### `NegatedEpicFilterInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `authorUsername` | [`String`](#string) | Filter by author username. | +| `labelName` | [`[String]`](#string) | Filter by label name. | +| `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | + ### `NegatedIssueFilterInput` #### Arguments diff --git a/doc/api/graphql/removed_items.md b/doc/api/graphql/removed_items.md index a76f1fb7418..d8fc6cb35f8 100644 --- a/doc/api/graphql/removed_items.md +++ b/doc/api/graphql/removed_items.md @@ -10,6 +10,32 @@ GraphQL is a versionless API, unlike the REST API. Occasionally, items have to be updated or removed from the GraphQL API. According to our [process for removing items](index.md#deprecation-and-removal-process), here are the items that have been removed. +## GitLab 14.0 + +Fields removed in [GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63293): + +### GraphQL Mutations + +| Argument name | Mutation | Deprecated in | Use instead | +| -------------------- | -------------------- | ------------- | -------------------------- | +| `updated_ids` | `todosMarkAllDone` | 13.2 | `todos` | +| `updated_ids` | `todoRestoreMany` | 13.2 | `todos` | +| `global_id` | `dastScannerProfileCreate`| 13.6 | `todos` | +| - | `addAwardEmoji` | 13.2 | `awardEmojiAdd` | +| - | `removeAwardEmoji` | 13.2 | `awardEmojiRemove` | +| - | `toggleAwardEmoji` | 13.2 | `ToggleAwardEmoji` | +| - | `runDastScan` | 13.5 | `dastOnDemandScanCreate` | +| - | `dismissVulnerability` | 13.5 | `vulnerabilityDismiss` | +| - | `revertVulnerabilityToDetected` | 13.5 | `vulnerabilityRevertToDetected` | + +### GraphQL Types + +| Field name | GraphQL type | Deprecated in | Use instead | +| -------------------- | -------------------- | ------------- | -------------------------- | +| `blob` | `SnippetType` | 13.3 | `blobs` | +| `global_id` | `DastScannerProfileType` | 13.6 | `blobs` | +| `vulnerabilities_count_by_day_and_severity` | `GroupType`, `QueryType` | 13.3 | None. Plaintext tokens no longer supported for security reasons. | + ## GitLab 13.6 Fields removed in [GitLab 13.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44866): diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index 848d5735096..63ba71797fc 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -102,7 +102,9 @@ POST /groups/:id/badges | `image_url` | string | yes | URL of the badge image | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" "https://gitlab.example.com/api/v4/groups/:id/badges" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&position=0" \ + "https://gitlab.example.com/api/v4/groups/:id/badges" ``` Example response: @@ -134,7 +136,8 @@ PUT /groups/:id/badges/:badge_id | `image_url` | string | no | URL of the badge image | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id" +curl --request PUT --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/:id/badges/:badge_id" ``` Example response: diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index ea7c13637c4..6853e38cc32 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -13,7 +13,7 @@ Similarly to [project-level](../user/project/clusters/index.md) and group-level Kubernetes clusters allow you to connect a Kubernetes cluster to your group, enabling you to use the same cluster across multiple projects. -Users need at least [Maintainer](../user/permissions.md) access for the group to use these endpoints. +Users need at least the [Maintainer role](../user/permissions.md) for the group to use these endpoints. ## List group clusters diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md index 79338df4f7a..d2bcac6332a 100644 --- a/doc/api/group_import_export.md +++ b/doc/api/group_import_export.md @@ -57,7 +57,8 @@ GET /groups/:id/export/download | `id` | integer/string | yes | ID of the group owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: " --remote-header-name --remote-name "https://gitlab.example.com/api/v4/groups/1/export/download" +curl --header "PRIVATE-TOKEN: " --remote-header-name \ + --remote-name "https://gitlab.example.com/api/v4/groups/1/export/download" ``` ```shell @@ -90,7 +91,9 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/groups/import" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "name=imported-group" --form "path=imported-group" \ + --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/groups/import" ``` NOTE: diff --git a/doc/api/group_labels.md b/doc/api/group_labels.md index 65c28e80f0a..2aca98259e0 100644 --- a/doc/api/group_labels.md +++ b/doc/api/group_labels.md @@ -118,7 +118,9 @@ POST /groups/:id/labels | `description` | string | no | The description of the label, | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"name": "Feature Proposal", "color": "#FFA500", "description": "Describes new ideas" }' "https://gitlab.example.com/api/v4/groups/5/labels" +curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"name": "Feature Proposal", "color": "#FFA500", "description": "Describes new ideas" }' \ + "https://gitlab.example.com/api/v4/groups/5/labels" ``` Example response: @@ -155,7 +157,8 @@ PUT /groups/:id/labels/:label_id | `description` | string | no | The description of the label. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" --data '{"new_name": "Feature Idea" }' "https://gitlab.example.com/api/v4/groups/5/labels/Feature%20Proposal" +curl --request PUT --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + --data '{"new_name": "Feature Idea" }' "https://gitlab.example.com/api/v4/groups/5/labels/Feature%20Proposal" ``` Example response: diff --git a/doc/api/group_level_variables.md b/doc/api/group_level_variables.md index b548372b02d..6425e022170 100644 --- a/doc/api/group_level_variables.md +++ b/doc/api/group_level_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -92,7 +92,8 @@ POST /groups/:id/variables | `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/variables/README.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/variables" --form "key=NEW_VARIABLE" --form "value=new value" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/1/variables" --form "key=NEW_VARIABLE" --form "value=new value" ``` ```json @@ -125,7 +126,8 @@ PUT /groups/:id/variables/:key | `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/variables/README.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/variables/NEW_VARIABLE" --form "value=updated value" +curl --request PUT --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/1/variables/NEW_VARIABLE" --form "value=updated value" ``` ```json @@ -153,5 +155,6 @@ DELETE /groups/:id/variables/:key | `key` | string | yes | The `key` of a variable | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/variables/VARIABLE_1" +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/1/variables/VARIABLE_1" ``` diff --git a/doc/api/group_protected_environments.md b/doc/api/group_protected_environments.md new file mode 100644 index 00000000000..d4e27a7200a --- /dev/null +++ b/doc/api/group_protected_environments.md @@ -0,0 +1,154 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: concepts, howto +--- + +# Group-level protected environments API **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215888) in [GitLab Premium](https://about.gitlab.com/pricing/) 14.0. +> - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - Disabled on GitLab.com. +> - Not recommended for production use. +> - To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](../ci/environments/protected_environments.md#enable-or-disable-group-level-protected-environments). **(FREE SELF)** + +This in-development feature might not be available for your use. There can be +[risks when enabling features still in development](../user/feature_flags.md#risks-when-enabling-features-still-in-development). +Refer to this feature's version history for more details. + +Read more about [group-level protected environments](../ci/environments/protected_environments.md#group-level-protected-environments), + +## Valid access levels + +The access levels are defined in the `ProtectedEnvironment::DeployAccessLevel::ALLOWED_ACCESS_LEVELS` method. +Currently, these levels are recognized: + +```plaintext +30 => Developer access +40 => Maintainer access +60 => Admin access +``` + +## List group-level protected environments + +Gets a list of protected environments from a group. + +```shell +GET /groups/:id/protected_environments +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) maintained by the authenticated user. | + +```shell +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/protected_environments/" +``` + +Example response: + +```json +[ + { + "name":"production", + "deploy_access_levels":[ + { + "access_level":40, + "access_level_description":"Maintainers", + "user_id":null, + "group_id":null + } + ] + } +] +``` + +## Get a single protected environment + +Gets a single protected environment. + +```shell +GET /groups/:id/protected_environments/:name +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) maintained by the authenticated user. | +| `name` | string | yes | The deployment tier of the protected environment. One of `production`, `staging`, `testing`, `development`, or `other`. Read more about [deployment tiers](../ci/environments/index.md#deployment-tier-of-environments).| + +```shell +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/protected_environments/production" +``` + +Example response: + +```json +{ + "name":"production", + "deploy_access_levels":[ + { + "access_level":40, + "access_level_description":"Maintainers", + "user_id":null, + "group_id":null + } + ] +} +``` + +## Protect an environment + +Protects a single environment. + +```shell +POST /groups/:id/protected_environments +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) maintained by the authenticated user. | +| `name` | string | yes | The deployment tier of the protected environment. One of `production`, `staging`, `testing`, `development`, or `other`. Read more about [deployment tiers](../ci/environments/index.md#deployment-tier-of-environments).| +| `deploy_access_levels` | array | yes | Array of access levels allowed to deploy, with each described by a hash. One of `user_id`, `group_id` or `access_level`. They take the form of `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}` respectively. | + +The assignable `user_id` are the users who belong to the given group with the Maintainer role (or above). +The assignable `group_id` are the sub-groups under the given group. + +```shell +curl --header 'Content-Type: application/json' --request POST --data '{"name": "production", "deploy_access_levels": [{"group_id": 9899826}]}' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/22034114/protected_environments" +``` + +Example response: + +```json +{ + "name":"production", + "deploy_access_levels":[ + { + "access_level":40, + "access_level_description":"protected-access-group", + "user_id":null, + "group_id":9899826 + } + ] +} +``` + +## Unprotect environment + +Unprotects the given protected environment. + +```shell +DELETE /groups/:id/protected_environments/:name +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) maintained by the authenticated user. | +| `name` | string | yes | The deployment tier of the protected environment. One of `production`, `staging`, `testing`, `development`, or `other`. Read more about [deployment tiers](../ci/environments/index.md#deployment-tier-of-environments).| + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/protected_environments/staging" +``` + +The response should return a 200 code. diff --git a/doc/api/group_relations_export.md b/doc/api/group_relations_export.md index bb19f7f0923..2f9c1e381df 100644 --- a/doc/api/group_relations_export.md +++ b/doc/api/group_relations_export.md @@ -48,7 +48,8 @@ GET /groups/:id/export_relations/status | `id` | integer/string | yes | ID of the group owned by the authenticated user. | ```shell -curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/export_relations/status" +curl --request GET --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/1/export_relations/status" ``` The status can be one of the following: @@ -92,7 +93,8 @@ GET /groups/:id/export_relations/download | `relation` | string | yes | Name of the group top-level relation to download. | ```shell -curl --header "PRIVATE-TOKEN: " --remote-header-name --remote-name "https://gitlab.example.com/api/v4/groups/1/export_relations/download?relation=labels" +curl --header "PRIVATE-TOKEN: " --remote-header-name \ + --remote-name "https://gitlab.example.com/api/v4/groups/1/export_relations/download?relation=labels" ``` ```shell diff --git a/doc/api/group_repository_storage_moves.md b/doc/api/group_repository_storage_moves.md index 0388bf46a1b..2373fa25e15 100644 --- a/doc/api/group_repository_storage_moves.md +++ b/doc/api/group_repository_storage_moves.md @@ -202,8 +202,10 @@ Supported attributes: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"destination_storage_name":"storage2"}' "https://gitlab.example.com/api/v4/groups/1/repository_storage_moves" +curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"destination_storage_name":"storage2"}' \ + "https://gitlab.example.com/api/v4/groups/1/repository_storage_moves" ``` Example response: @@ -241,8 +243,10 @@ Supported attributes: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"source_storage_name":"default"}' "https://gitlab.example.com/api/v4/group_repository_storage_moves" +curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"default"}' \ + "https://gitlab.example.com/api/v4/group_repository_storage_moves" ``` Example response: diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index f0c38d4d4b9..58192425786 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -64,7 +64,7 @@ GET /groups/:id/wikis/:slug | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `slug` | string | yes | The slug (a unique string) of the wiki page | +| `slug` | string | yes | URL-encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/wikis/home" @@ -127,7 +127,7 @@ PUT /groups/:id/wikis/:slug | `content` | string | yes if `title` is not provided | The content of the wiki page | | `title` | string | yes if `content` is not provided | The title of the wiki page | | `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | -| `slug` | string | yes | The slug (a unique identifier) of the wiki page | +| `slug` | string | yes | URL encoded slug (a unique string) of the wiki page. Ex. dir%2Fpage_name | ```shell curl --request PUT --data "format=rdoc&content=documentation&title=Docs" \ @@ -157,7 +157,7 @@ DELETE /groups/:id/wikis/:slug | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `slug` | string | yes | The slug (a unique identifier) of the wiki page | +| `slug` | string | yes | URL-encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/1/wikis/foo" @@ -186,7 +186,8 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "file=@dk.png" "https://gitlab.example.com/api/v4/groups/1/wikis/attachments" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "file=@dk.png" "https://gitlab.example.com/api/v4/groups/1/wikis/attachments" ``` Example response: diff --git a/doc/api/groups.md b/doc/api/groups.md index 6bec6e0f6f8..de2c6c95bcd 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -302,7 +302,8 @@ Example response: "id": 9, "description": "foo", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "archived": false, "visibility": "internal", "ssh_url_to_repo": "git@gitlab.example.com/html5-boilerplate.git", @@ -381,9 +382,8 @@ Example response: "path_with_namespace":"h5bp/html5-boilerplate", "created_at":"2020-04-27T06:13:22.642Z", "default_branch":"master", - "tag_list":[ - - ], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo":"ssh://git@gitlab.com/h5bp/html5-boilerplate.git", "http_url_to_repo":"http://gitlab.com/h5bp/html5-boilerplate.git", "web_url":"http://gitlab.com/h5bp/html5-boilerplate", @@ -540,7 +540,8 @@ Example response: "id": 7, "description": "Voluptas veniam qui et beatae voluptas doloremque explicabo facilis.", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "archived": false, "visibility": "public", "ssh_url_to_repo": "git@gitlab.example.com:twitter/typeahead-js.git", @@ -578,7 +579,8 @@ Example response: "id": 6, "description": "Aspernatur omnis repudiandae qui voluptatibus eaque.", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "archived": false, "visibility": "internal", "ssh_url_to_repo": "git@gitlab.example.com:twitter/flight.git", @@ -618,7 +620,8 @@ Example response: "id": 8, "description": "Velit eveniet provident fugiat saepe eligendi autem.", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "archived": false, "visibility": "private", "ssh_url_to_repo": "git@gitlab.example.com:h5bp/html5-boilerplate.git", @@ -722,6 +725,28 @@ Example response: } ``` +### Download a Group avatar + +Get a group avatar. This endpoint can be accessed without authentication if the +group is publicly accessible. + +```plaintext +GET /groups/:id/avatar +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | --------------------- | +| `id` | integer/string | yes | ID of the group | + +Example: + +```shell +curl --header "PRIVATE-TOKEN: $GITLAB_LOCAL_TOKEN" \ + --remote-header-name \ + --remote-name \ + "https://gitlab.example.com/api/v4/groups/4/avatar" +``` + ### Disable the results limit **(FREE SELF)** The 100 results limit can break integrations developed using GitLab 12.4 and earlier. @@ -752,7 +777,7 @@ Parameters: | `name` | string | yes | The name of the group. | | `path` | string | yes | The path of the group. | | `description` | string | no | The group's description. | -| `membership_lock` | boolean | no | **(STARTER)** Prevent adding new members to project membership within this group. | +| `membership_lock` | boolean | no | **(PREMIUM)** Prevent adding new members to project membership within this group. | | `visibility` | string | no | The group's visibility. Can be `private`, `internal`, or `public`. | | `share_with_group_lock` | boolean | no | Prevent sharing a project with another group within this group. | | `require_two_factor_authentication` | boolean | no | Require all users in this group to setup Two-factor authentication. | @@ -770,6 +795,10 @@ Parameters: | `shared_runners_minutes_limit` | integer | no | **(PREMIUM SELF)** Pipeline minutes quota for this group (included in plan). Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0` | | `extra_shared_runners_minutes_limit` | integer | no | **(PREMIUM SELF)** Extra pipeline minutes quota for this group (purchased in addition to the minutes included in the plan). | +NOTE: +On GitLab SaaS, you must use the GitLab UI to create groups without a parent group. You cannot +use the API to do this. + ### Options for `default_branch_protection` The `default_branch_protection` attribute determines whether developers and maintainers can push to the applicable [default branch](../user/project/repository/branches/default.md), as described in the following table: @@ -788,9 +817,10 @@ This is similar to creating a [New group](#new-group). You need the `parent_id` - `subgroup_name` ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ - --data '{"path": "", "name": "", "parent_id": }' \ - "https://gitlab.example.com/api/v4/groups/" +curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"path": "", "name": "", "parent_id": }' \ + "https://gitlab.example.com/api/v4/groups/" ``` ## Transfer project to group @@ -809,7 +839,8 @@ Parameters: | `project_id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/4/projects/56" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/4/projects/56" ``` ## Update group @@ -826,7 +857,7 @@ PUT /groups/:id | `name` | string | no | The name of the group. | | `path` | string | no | The path of the group. | | `description` | string | no | The description of the group. | -| `membership_lock` | boolean | no | **(STARTER)** Prevent adding new members to project membership within this group. | +| `membership_lock` | boolean | no | **(PREMIUM)** Prevent adding new members to project membership within this group. | | `share_with_group_lock` | boolean | no | Prevent sharing a project with another group within this group. | | `visibility` | string | no | The visibility level of the group. Can be `private`, `internal`, or `public`. | | `require_two_factor_authentication` | boolean | no | Require all users in this group to setup Two-factor authentication. | @@ -851,7 +882,8 @@ The `projects` and `shared_projects` attributes in the response are deprecated a To get the details of all projects within a group, use either the [list a group's projects](#list-a-groups-projects) or the [list a group's shared projects](#list-a-groups-shared-projects) endpoint. ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5?name=Experimental" +curl --request PUT --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5?name=Experimental" ``` This endpoint returns: @@ -883,7 +915,8 @@ Example response: "id": 9, "description": "foo", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "public": false, "archived": false, "visibility": "internal", @@ -956,7 +989,8 @@ curl to post data using the header `Content-Type: multipart/form-data`. The `@`. For example: ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/22" --form "avatar=@/tmp/example.png" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/22" \ + --form "avatar=@/tmp/example.png" ``` ## Remove group @@ -1142,7 +1176,7 @@ DELETE /groups/:id/hooks/:hook_id | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `hook_id` | integer | yes | The ID of the group hook. | -## Group Audit Events **(STARTER)** +## Group Audit Events **(PREMIUM)** Group audit events can be accessed via the [Group Audit Events API](audit_events.md#group-audit-events) @@ -1298,11 +1332,11 @@ DELETE /groups/:id/share/:group_id | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | | `group_id` | integer | yes | The ID of the group to share with | -## Push Rules **(STARTER)** +## Push Rules **(PREMIUM)** -> Introduced in [GitLab Starter](https://about.gitlab.com/pricing/) 13.4. +> Introduced in [GitLab](https://about.gitlab.com/pricing/) 13.4. -### Get group push rules **(STARTER)** +### Get group push rules **(PREMIUM)** Get the [push rules](../user/group/index.md#group-push-rules) of a group. @@ -1345,7 +1379,7 @@ the `commit_committer_check` and `reject_unsigned_commits` parameters: } ``` -### Add group push rule **(STARTER)** +### Add group push rule **(PREMIUM)** Adds [push rules](../user/group/index.md#group-push-rules) to the specified group. @@ -1358,17 +1392,17 @@ POST /groups/:id/push_rule | Attribute | Type | Required | Description | | --------------------------------------------- | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `deny_delete_tag` **(STARTER)** | boolean | no | Deny deleting a tag | -| `member_check` **(STARTER)** | boolean | no | Allows only GitLab users to author commits | -| `prevent_secrets` **(STARTER)** | boolean | no | [Files that are likely to contain secrets](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml) are rejected | -| `commit_message_regex` **(STARTER)** | string | no | All commit messages must match the regular expression provided in this attribute, e.g. `Fixed \d+\..*` | -| `commit_message_negative_regex` **(STARTER)** | string | no | Commit messages matching the regular expression provided in this attribute aren't allowed, e.g. `ssh\:\/\/` | -| `branch_name_regex` **(STARTER)** | string | no | All branch names must match the regular expression provided in this attribute, e.g. `(feature|hotfix)\/*` | -| `author_email_regex` **(STARTER)** | string | no | All commit author emails must match the regular expression provided in this attribute, e.g. `@my-company.com$` | -| `file_name_regex` **(STARTER)** | string | no | Filenames matching the regular expression provided in this attribute are **not** allowed, e.g. `(jar|exe)$` | -| `max_file_size` **(STARTER)** | integer | no | Maximum file size (MB) allowed | -| `commit_committer_check` **(PREMIUM)** | boolean | no | Only commits pushed using verified emails are allowed | -| `reject_unsigned_commits` **(PREMIUM)** | boolean | no | Only commits signed through GPG are allowed | +| `deny_delete_tag` | boolean | no | Deny deleting a tag | +| `member_check` | boolean | no | Allows only GitLab users to author commits | +| `prevent_secrets` | boolean | no | [Files that are likely to contain secrets](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml) are rejected | +| `commit_message_regex` | string | no | All commit messages must match the regular expression provided in this attribute, e.g. `Fixed \d+\..*` | +| `commit_message_negative_regex` | string | no | Commit messages matching the regular expression provided in this attribute aren't allowed, e.g. `ssh\:\/\/` | +| `branch_name_regex` | string | no | All branch names must match the regular expression provided in this attribute, e.g. `(feature|hotfix)\/*` | +| `author_email_regex` | string | no | All commit author emails must match the regular expression provided in this attribute, e.g. `@my-company.com$` | +| `file_name_regex` | string | no | Filenames matching the regular expression provided in this attribute are **not** allowed, e.g. `(jar|exe)$` | +| `max_file_size` | integer | no | Maximum file size (MB) allowed | +| `commit_committer_check` | boolean | no | Only commits pushed using verified emails are allowed | +| `reject_unsigned_commits` | boolean | no | Only commits signed through GPG are allowed | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/19/push_rule" @@ -1392,7 +1426,7 @@ Response: } ``` -### Edit group push rule **(STARTER)** +### Edit group push rule **(PREMIUM)** Edit push rules for a specified group. @@ -1405,17 +1439,17 @@ PUT /groups/:id/push_rule | Attribute | Type | Required | Description | | --------------------------------------------- | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `deny_delete_tag` **(STARTER)** | boolean | no | Deny deleting a tag | -| `member_check` **(STARTER)** | boolean | no | Restricts commits to be authored by existing GitLab users only | -| `prevent_secrets` **(STARTER)** | boolean | no | [Files that are likely to contain secrets](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml) are rejected | -| `commit_message_regex` **(STARTER)** | string | no | All commit messages must match the regular expression provided in this attribute, e.g. `Fixed \d+\..*` | -| `commit_message_negative_regex` **(STARTER)** | string | no | Commit messages matching the regular expression provided in this attribute aren't allowed, e.g. `ssh\:\/\/` | -| `branch_name_regex` **(STARTER)** | string | no | All branch names must match the regular expression provided in this attribute, e.g. `(feature|hotfix)\/*` | -| `author_email_regex` **(STARTER)** | string | no | All commit author emails must match the regular expression provided in this attribute, e.g. `@my-company.com$` | -| `file_name_regex` **(STARTER)** | string | no | Filenames matching the regular expression provided in this attribute are **not** allowed, e.g. `(jar|exe)$` | -| `max_file_size` **(STARTER)** | integer | no | Maximum file size (MB) allowed | -| `commit_committer_check` **(PREMIUM)** | boolean | no | Only commits pushed using verified emails are allowed | -| `reject_unsigned_commits` **(PREMIUM)** | boolean | no | Only commits signed through GPG are allowed | +| `deny_delete_tag` | boolean | no | Deny deleting a tag | +| `member_check` | boolean | no | Restricts commits to be authored by existing GitLab users only | +| `prevent_secrets` | boolean | no | [Files that are likely to contain secrets](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml) are rejected | +| `commit_message_regex` | string | no | All commit messages must match the regular expression provided in this attribute, e.g. `Fixed \d+\..*` | +| `commit_message_negative_regex` | string | no | Commit messages matching the regular expression provided in this attribute aren't allowed, e.g. `ssh\:\/\/` | +| `branch_name_regex` | string | no | All branch names must match the regular expression provided in this attribute, e.g. `(feature|hotfix)\/*` | +| `author_email_regex` | string | no | All commit author emails must match the regular expression provided in this attribute, e.g. `@my-company.com$` | +| `file_name_regex` | string | no | Filenames matching the regular expression provided in this attribute are **not** allowed, e.g. `(jar|exe)$` | +| `max_file_size` | integer | no | Maximum file size (MB) allowed | +| `commit_committer_check` | boolean | no | Only commits pushed using verified emails are allowed | +| `reject_unsigned_commits` | boolean | no | Only commits signed through GPG are allowed | ```shell curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/19/push_rule" @@ -1439,7 +1473,7 @@ Response: } ``` -### Delete group push rule **(STARTER)** +### Delete group push rule **(PREMIUM)** Deletes the [push rules](../user/group/index.md#group-push-rules) of a group. diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index 58e69e22a15..de6fd958aa6 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -85,7 +85,8 @@ POST /admin/ci/variables | `masked` | boolean | no | Whether the variable is masked. | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/admin/ci/variables" --form "key=NEW_VARIABLE" --form "value=new value" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/admin/ci/variables" --form "key=NEW_VARIABLE" --form "value=new value" ``` ```json @@ -115,7 +116,8 @@ PUT /admin/ci/variables/:key | `masked` | boolean | no | Whether the variable is masked. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/admin/ci/variables/NEW_VARIABLE" --form "value=updated value" +curl --request PUT --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/admin/ci/variables/NEW_VARIABLE" --form "value=updated value" ``` ```json diff --git a/doc/api/invitations.md b/doc/api/invitations.md index fbdecd0e3fa..36ff2d5bda4 100644 --- a/doc/api/invitations.md +++ b/doc/api/invitations.md @@ -4,7 +4,7 @@ group: Expansion info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Invitations API +# Invitations API **(FREE)** Use the Invitations API to send email to users you want to join a group or project, and to list pending invitations. @@ -41,10 +41,13 @@ POST /projects/:id/invitations | `email` | string | yes | The email of the new member or multiple emails separated by commas | | `access_level` | integer | yes | A valid access level | | `expires_at` | string | no | A date string in the format YEAR-MONTH-DAY | +| `invite_source` | string | no | The source of the invitation that starts the member creation process. See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/327120). | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "email=test@example.com&access_level=30" "https://gitlab.example.com/api/v4/groups/:id/invitations" -curl --request POST --header "PRIVATE-TOKEN: " --data "email=test@example.com&access_level=30" "https://gitlab.example.com/api/v4/projects/:id/invitations" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "email=test@example.com&access_level=30" "https://gitlab.example.com/api/v4/groups/:id/invitations" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "email=test@example.com&access_level=30" "https://gitlab.example.com/api/v4/projects/:id/invitations" ``` Example responses: diff --git a/doc/api/issues.md b/doc/api/issues.md index acfca50cb5e..f321c00e7f2 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -65,7 +65,6 @@ GET /issues?state=opened | `issue_type` | string | no | Filter to a given type of issue. One of `issue`, `incident`, or `test_case`. _(Introduced in [GitLab 13.12](https://gitlab.com/gitlab-org/gitlab/-/issues/260375))_ | | `iteration_id` **(PREMIUM)** | integer | no | Return issues assigned to the given iteration ID. `None` returns issues that do not belong to an iteration. `Any` returns issues that belong to an iteration. Mutually exclusive with `iteration_title`. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in GitLab 13.6)_ | | `iteration_title` **(PREMIUM)** | string | no | Return issues assigned to the iteration with the given title. Similar to `iteration_id` and mutually exclusive with `iteration_id`. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in GitLab 13.6)_ | -| `milestone` | string | no | The milestone title. `None` lists all issues with no milestone. `Any` lists all issues that have an assigned milestone. | | `labels` | string | no | Comma-separated list of label names, issues must have all labels to be returned. `None` lists all issues with no labels. `Any` lists all issues with at least one label. `No+Label` (Deprecated) lists all issues with no labels. Predefined names are case-insensitive. | | `milestone` | string | no | The milestone title. `None` lists all issues with no milestone. `Any` lists all issues that have an assigned milestone. | | `my_reaction_emoji` | string | no | Return issues reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. _([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14016) in GitLab 10.0)_ | @@ -996,7 +995,7 @@ POST /projects/:id/issues | `issue_type` | string | no | The type of issue. One of `issue`, `incident`, or `test_case`. Default is `issue`. | | `labels` | string | no | Comma-separated label names for an issue | | `merge_request_to_resolve_discussions_of` | integer | no | The IID of a merge request in which to resolve all issues. This fills out the issue with a default description and mark all discussions as resolved. When passing a description or title, these values take precedence over the default values.| -| `milestone_id` | integer | no | The global ID of a milestone to assign issue | +| `milestone_id` | integer | no | The global ID of a milestone to assign issue. To find the `milestone_id` associated with a milestone, view an issue with the milestone assigned and [use the API](#single-project-issue) to retrieve the issue's details. | | `title` | string | yes | The title of an issue | | `weight` **(PREMIUM)** | integer | no | The weight of the issue. Valid values are greater than or equal to 0. | @@ -2081,6 +2080,7 @@ Example response: "source_project_id": 1, "target_project_id": 1, "labels": [], + "draft": false, "work_in_progress": false, "milestone": { "id": 27, @@ -2232,6 +2232,7 @@ Example response: "closed_at": null, "closed_by": null, "labels": [], + "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index 0dbb35a62cd..54404559577 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Testing info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -90,7 +90,7 @@ Parameters Example request using the `PRIVATE-TOKEN` header: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/download?job=test" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/main/download?job=test" ``` To use this in a [`script` definition](../ci/yaml/README.md#script) inside @@ -98,25 +98,25 @@ To use this in a [`script` definition](../ci/yaml/README.md#script) inside - The `JOB-TOKEN` header with the GitLab-provided `CI_JOB_TOKEN` variable. For example, the following job downloads the artifacts of the `test` job - of the `master` branch. Note that the command is wrapped into single quotes + of the `main` branch. Note that the command is wrapped into single quotes because it contains a colon (`:`): ```yaml artifact_download: stage: test script: - - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test"' + - 'curl --location --output artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/main/download?job=test"' ``` - Or the `job_token` attribute with the GitLab-provided `CI_JOB_TOKEN` variable. For example, the following job downloads the artifacts of the `test` job - of the `master` branch: + of the `main` branch: ```yaml artifact_download: stage: test script: - - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/master/download?job=test&job_token=$CI_JOB_TOKEN"' + - 'curl --location --output artifacts.zip "https://gitlab.example.com/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/main/download?job=test&job_token=$CI_JOB_TOKEN"' ``` Possible response status codes: @@ -193,7 +193,7 @@ Parameters: Example request: ```shell -curl --location --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/master/raw/some/release/file.pdf?job=pdf" +curl --location --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/jobs/artifacts/main/raw/some/release/file.pdf?job=pdf" ``` Possible response status codes: @@ -243,7 +243,7 @@ Example response: "download_url": null, "id": 42, "name": "rubocop", - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 6647b53bcb4..b92f2a72c03 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -1,10 +1,10 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Jobs API +# Jobs API **(FREE)** ## List project jobs @@ -63,11 +63,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "runner": null, "stage": "test", "status": "failed", @@ -117,11 +117,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -198,11 +198,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -263,11 +263,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "runner": null, "stage": "test", "status": "failed", @@ -348,14 +348,14 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending", "created_at": "2015-12-24T15:50:16.123Z", "updated_at": "2015-12-24T18:00:44.432Z", "web_url": "https://example.com/foo/bar/pipelines/6" }, - "ref": "master", + "ref": "main", "stage": "test", "status": "pending", "tag": false, @@ -380,7 +380,7 @@ Example of response "downstream_pipeline": { "id": 5, "sha": "f62a4b2fb89754372a346f24659212eb8da13601", - "ref": "master", + "ref": "main", "status": "pending", "created_at": "2015-12-24T17:54:27.722Z", "updated_at": "2015-12-24T17:58:27.896Z", @@ -433,11 +433,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -518,7 +518,7 @@ Example response: "id": 1, "project_id": 1, "sha": "b83d6e391c22777fca1ed3012fce84f633d7fed0", - "ref": "master", + "ref": "main", "status": "pending", "created_at": "2021-03-26T14:51:51.107Z", "updated_at": "2021-03-26T14:51:51.107Z", @@ -590,11 +590,11 @@ Example of response "pipeline": { "id": 6, "project_id": 1, - "ref": "master", + "ref": "main", "sha": "0ff3ae198f8601a285adcf5c0fff204ee6fba5fd", "status": "pending" }, - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -684,7 +684,7 @@ Example of response "queued_duration": 0.010, "id": 42, "name": "rubocop", - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -734,7 +734,7 @@ Example of response "queued_duration": 0.010, "id": 42, "name": "rubocop", - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -784,7 +784,7 @@ Example of response "download_url": null, "id": 42, "name": "rubocop", - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", @@ -839,7 +839,7 @@ Example of response "queued_duration": 0.010, "id": 42, "name": "rubocop", - "ref": "master", + "ref": "main", "artifacts": [], "runner": null, "stage": "test", diff --git a/doc/api/labels.md b/doc/api/labels.md index a9f2698a270..5abab7a79c4 100644 --- a/doc/api/labels.md +++ b/doc/api/labels.md @@ -46,8 +46,7 @@ Example response: "open_merge_requests_count": 1, "subscribed": false, "priority": 10, - "is_project_label": true, - "remove_on_close": false + "is_project_label": true }, { "id" : 4, @@ -61,8 +60,7 @@ Example response: "open_merge_requests_count": 0, "subscribed": false, "priority": null, - "is_project_label": true, - "remove_on_close": false + "is_project_label": true }, { "id" : 7, @@ -76,8 +74,7 @@ Example response: "open_merge_requests_count": 1, "subscribed": false, "priority": null, - "is_project_label": true, - "remove_on_close": true + "is_project_label": true }, { "id" : 8, @@ -91,8 +88,7 @@ Example response: "open_merge_requests_count": 2, "subscribed": false, "priority": null, - "is_project_label": false, - "remove_on_close": false + "is_project_label": false }, { "id" : 9, @@ -106,8 +102,7 @@ Example response: "open_merge_requests_count": 1, "subscribed": true, "priority": null, - "is_project_label": true, - "remove_on_close": false + "is_project_label": true } ] ``` @@ -145,8 +140,7 @@ Example response: "open_merge_requests_count": 1, "subscribed": false, "priority": 10, - "is_project_label": true, - "remove_on_close": true + "is_project_label": true } ``` @@ -165,7 +159,6 @@ POST /projects/:id/labels | `color` | string | yes | The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the [CSS color names](https://developer.mozilla.org/en-US/docs/Web/CSS/color_value#Color_keywords) | | `description` | string | no | The description of the label | | `priority` | integer | no | The priority of the label. Must be greater or equal than zero or `null` to remove the priority. | -| `remove_on_close` | boolean | no | Whether the label should be removed from an issue when the issue is closed. _([Introduced in GitLab 13.12](https://gitlab.com/gitlab-org/gitlab/-/issues/17461))_ | ```shell curl --data "name=feature&color=#5843AD" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/labels" @@ -186,8 +179,7 @@ Example response: "open_merge_requests_count": 0, "subscribed": false, "priority": null, - "is_project_label": true, - "remove_on_close": true + "is_project_label": true } ``` @@ -228,10 +220,10 @@ PUT /projects/:id/labels/:label_id | `color` | string | yes if `new_name` is not provided | The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the [CSS color names](https://developer.mozilla.org/en-US/docs/Web/CSS/color_value#Color_keywords) | | `description` | string | no | The new description of the label | | `priority` | integer | no | The new priority of the label. Must be greater or equal than zero or `null` to remove the priority. | -| `remove_on_close` | boolean | no | Boolean option specifying whether the label should be removed from issues when they are closed. | ```shell -curl --request PUT --data "new_name=docs&color=#8E44AD&description=Documentation" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/labels/documentation" +curl --request PUT --data "new_name=docs&color=#8E44AD&description=Documentation" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/labels/documentation" ``` Example response: @@ -249,8 +241,7 @@ Example response: "open_merge_requests_count": 2, "subscribed": false, "priority": null, - "is_project_label": true, - "remove_on_close": true + "is_project_label": true } ``` @@ -291,8 +282,7 @@ Example response: "open_issues_count": 1, "closed_issues_count": 0, "open_merge_requests_count": 2, - "subscribed": false, - "remove_on_close": true + "subscribed": false } ``` @@ -333,8 +323,7 @@ Example response: "open_merge_requests_count": 1, "subscribed": true, "priority": null, - "is_project_label": true, - "remove_on_close": true + "is_project_label": true } ``` diff --git a/doc/api/lint.md b/doc/api/lint.md index 867a5e54663..57d11d15adc 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -1,19 +1,26 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# CI Lint API +# CI Lint API **(FREE)** ## Validate the CI YAML configuration -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5953) in GitLab 8.12. - Checks if CI/CD YAML configuration is valid. This endpoint validates basic CI/CD configuration syntax. It doesn't have any namespace specific context. -Access to this endpoint requires authentication. +Access to this endpoint does not require authentication when the instance +[allows new sign ups](../user/admin_area/settings/sign_up_restrictions.md#disable-new-sign-ups) +and: + +- Does not have an [allowlist or denylist](../user/admin_area/settings/sign_up_restrictions.md#allow-or-deny-sign-ups-using-specific-email-domains). +- Does not [require administrator approval for new sign ups](../user/admin_area/settings/sign_up_restrictions.md#require-administrator-approval-for-new-sign-ups). +- Does not have additional [sign up + restrictions](../user/admin_area/settings/sign_up_restrictions.html#sign-up-restrictions). + +Otherwise, authentication is required. ```plaintext POST /ci/lint @@ -111,7 +118,7 @@ Example response: { "status": "valid", "errors": [], - "merged_config": "---\n:another_test:\n :stage: test\n :script: echo 2\n:test:\n :stage: test\n :script: echo 1\n" + "merged_yaml": "---\n:another_test:\n :stage: test\n :script: echo 2\n:test:\n :stage: test\n :script: echo 1\n" } ``` diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index 68a19b9912f..4be4f83b4ce 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -128,7 +128,8 @@ PATCH /projects/:id/managed_licenses/:managed_license_id | `approval_status` | string | yes | The approval status. "approved" or "blacklisted" | ```shell -curl --request PATCH --data "approval_status=blacklisted" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses/6" +curl --request PATCH --data "approval_status=blacklisted" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses/6" ``` Example response: diff --git a/doc/api/members.md b/doc/api/members.md index 2a70e35b287..627c9a12b5e 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -415,10 +415,13 @@ POST /projects/:id/members | `user_id` | integer/string | yes | The user ID of the new member or multiple IDs separated by commas | | `access_level` | integer | yes | A valid access level | | `expires_at` | string | no | A date string in the format `YEAR-MONTH-DAY` | +| `invite_source` | string | no | The source of the invitation that starts the member creation process. See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/327120). | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/groups/:id/members" -curl --request POST --header "PRIVATE-TOKEN: " --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/projects/:id/members" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/groups/:id/members" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "user_id=1&access_level=30" "https://gitlab.example.com/api/v4/projects/:id/members" ``` Example response: diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 978cbff625c..8947e5b382f 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -29,7 +29,7 @@ GET /projects/:id/approvals | Attribute | Type | Required | Description | | --------- | ------- | -------- | ------------------- | -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | ```json { @@ -58,7 +58,7 @@ POST /projects/:id/approvals | Attribute | Type | Required | Description | | ------------------------------------------------ | ------- | -------- | --------------------------------------------------------------------------------------------------- | -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `approvals_before_merge` | integer | no | How many approvals are required before an MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | | `reset_approvals_on_push` | boolean | no | Reset approvals on a new push | | `disable_overriding_approvers_per_merge_request` | boolean | no | Allow/Disallow overriding approvers per MR | @@ -93,7 +93,7 @@ GET /projects/:id/approval_rules | Attribute | Type | Required | Description | |----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | ```json [ @@ -192,7 +192,7 @@ GET /projects/:id/approval_rules/:approval_rule_id | Attribute | Type | Required | Description | |----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `approval_rule_id` | integer | yes | The ID of a approval rule | ```json @@ -291,7 +291,7 @@ POST /projects/:id/approval_rules | Attribute | Type | Required | Description | |------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `name` | string | yes | The name of the approval rule | | `approvals_required` | integer | yes | The number of required approvals for this rule | | `user_ids` | Array | no | The ids of users as approvers | @@ -396,7 +396,7 @@ PUT /projects/:id/approval_rules/:approval_rule_id | Attribute | Type | Required | Description | |------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `approval_rule_id` | integer | yes | The ID of a approval rule | | `name` | string | yes | The name of the approval rule | | `approvals_required` | integer | yes | The number of required approvals for this rule | @@ -500,123 +500,9 @@ DELETE /projects/:id/approval_rules/:approval_rule_id | Attribute | Type | Required | Description | |----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `approval_rule_id` | integer | yes | The ID of a approval rule -## External Project-level MR approvals **(ULTIMATE)** - -Configuration for approvals on a specific Merge Request which makes a call to an external HTTP resource. - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3869) in GitLab 13.10. -> - It's [deployed behind a feature flag](../user/feature_flags.md), disabled by default. -> - It's disabled on GitLab.com. -> - It's not recommended for production use. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-external-project-level-mr-approvals). **(ULTIMATE SELF)** - -### Get project external approval rules **(ULTIMATE)** - -You can request information about a project's external approval rules using the following endpoint: - -```plaintext -GET /projects/:id/external_approval_rules -``` - -**Parameters:** - -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a project | - -```json -[ - { - "id": 1, - "name": "Compliance Check", - "project_id": 6, - "external_url": "https://gitlab.com/example/test.json", - "protected_branches": [ - { - "id": 14, - "project_id": 6, - "name": "master", - "created_at": "2020-10-12T14:04:50.787Z", - "updated_at": "2020-10-12T14:04:50.787Z", - "code_owner_approval_required": false - } - ] - } -] -``` - -### Create external approval rule **(ULTIMATE)** - -You can create a new external approval rule for a project using the following endpoint: - -```plaintext -POST /projects/:id/external_approval_rules -``` - -| Attribute | Type | Required | Description | -|------------------------|----------------|----------|----------------------------------------------------| -| `id` | integer | yes | The ID of a project | -| `name` | string | yes | Display name of approval rule | -| `external_url` | string | yes | URL of external approval resource | -| `protected_branch_ids` | `array` | no | The ids of protected branches to scope the rule by | - -### Delete external approval rule **(ULTIMATE)** - -You can delete an external approval rule for a project using the following endpoint: - -```plaintext -DELETE /projects/:id/external_approval_rules/:rule_id -``` - -| Attribute | Type | Required | Description | -|------------------------|----------------|----------|----------------------------------------------------| -| `rule_id` | integer | yes | The ID of an approval rule | -| `id` | integer | yes | The ID of a project | - -### Update external approval rule **(ULTIMATE)** - -You can update an existing external approval rule for a project using the following endpoint: - -```plaintext -PUT /projects/:id/external_approval_rules/:rule_id -``` - -| Attribute | Type | Required | Description | -|------------------------|----------------|----------|----------------------------------------------------| -| `id` | integer | yes | The ID of a project | -| `rule_id` | integer | yes | The ID of an external approval rule | -| `name` | string | no | Display name of approval rule | -| `external_url` | string | no | URL of external approval resource | -| `protected_branch_ids` | `array` | no | The ids of protected branches to scope the rule by | - -### Enable or disable External Project-level MR approvals **(ULTIMATE SELF)** - -Enable or disable External Project-level MR approvals is under development and not ready for production use. It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](../user/feature_flags.md) -can enable it. - -To enable it: - -```ruby -# For the instance -Feature.enable(:ff_compliance_approval_gates) -# For a single project -Feature.enable(:ff_compliance_approval_gates, Project.find()) -``` - -To disable it: - -```ruby -# For the instance -Feature.disable(:ff_compliance_approval_gates) -# For a single project -Feature.disable(:ff_compliance_approval_gates, Project.find()) -``` - ## Merge Request-level MR approvals Configuration for approvals on a specific Merge Request. Must be authenticated for all endpoints. @@ -637,7 +523,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/approvals | Attribute | Type | Required | Description | |---------------------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The IID of MR | ```json @@ -684,7 +570,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals | Attribute | Type | Required | Description | |----------------------|---------|----------|--------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The IID of MR | | `approvals_required` | integer | yes | Approvals required before MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | @@ -726,7 +612,7 @@ This includes additional information about the users who have already approved | Attribute | Type | Required | Description | |----------------------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The IID of MR | ```json @@ -793,7 +679,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/approval_rules | Attribute | Type | Required | Description | |---------------------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The IID of MR | ```json @@ -871,7 +757,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/approval_rules | Attribute | Type | Required | Description | |----------------------------|---------|----------|------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The IID of MR | | `name` | string | yes | The name of the approval rule | | `approvals_required` | integer | yes | The number of required approvals for this rule | @@ -961,7 +847,7 @@ These are system generated rules. | Attribute | Type | Required | Description | |----------------------|---------|----------|------------------------------------------------| -| `id` | integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | | `merge_request_iid` | integer | yes | The ID of MR | | `approval_rule_id` | integer | yes | The ID of a approval rule | | `name` | string | yes | The name of the approval rule | @@ -1045,9 +931,9 @@ These are system generated rules. | Attribute | Type | Required | Description | |---------------------|---------|----------|---------------------------| -| `id` | integer | yes | The ID of a project | -| `merge_request_iid` | integer | yes | The ID of MR | -| `approval_rule_id` | integer | yes | The ID of a approval rule | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of the merge request | +| `approval_rule_id` | integer | yes | The ID of an approval rule | ## Approve Merge Request @@ -1065,9 +951,9 @@ POST /projects/:id/merge_requests/:merge_request_iid/approve | Attribute | Type | Required | Description | |---------------------|---------|----------|-------------------------| -| `id` | integer | yes | The ID of a project | -| `merge_request_iid` | integer | yes | The IID of MR | -| `sha` | string | no | The HEAD of the MR | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of the merge request | +| `sha` | string | no | The `HEAD` of the merge request | | `approval_password` **(PREMIUM)** | string | no | Current user's password. Required if [**Require user password to approve**](../user/project/merge_requests/approvals/settings.md#require-authentication-for-approvals) is enabled in the project settings. | The `sha` parameter works in the same way as @@ -1129,5 +1015,5 @@ POST /projects/:id/merge_requests/:merge_request_iid/unapprove | Attribute | Type | Required | Description | |---------------------|---------|----------|---------------------| -| `id` | integer | yes | The ID of a project | -| `merge_request_iid` | integer | yes | The IID of MR | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of a merge request | diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index c4cb7753fc9..57754f62d5a 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -16,6 +16,7 @@ type: reference, api > - `with_merge_status_recheck` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. > - `reviewer_username` and `reviewer_id` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. > - `reviewer_ids` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51186) in GitLab 13.8. +> - `draft` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63473) as an eventual replacement for `work_in_progress` in GitLab 14.0 Every API call to merge requests must be authenticated. @@ -174,6 +175,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -357,6 +359,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -405,6 +408,16 @@ Parameters: ] ``` +The `merge_status` field may hold one of the following values: + +| Value | Interpretation | +|----------------------------|-----------------------------------------------------------------------| +| `unchecked` | We have not checked this yet | +| `checking` | We are currently checking if the merge request can be merged | +| `can_be_merged` | This merge request can be merged without conflict | +| `cannot_be_merged` | There are merge conflicts between the source and target branches | +| `cannot_be_merged_recheck` | Currently unchecked. Before the current changes, there were conflicts | + Users on GitLab Premium or higher also see the `approvals_before_merge` parameter: @@ -532,6 +545,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -670,6 +684,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -902,6 +917,7 @@ Parameters: "target_project_id": 4, "labels": [ ], "description": "Qui voluptatibus placeat ipsa alias quasi. Deleniti rem ut sint. Optio velit qui distinctio.", + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -1112,6 +1128,7 @@ POST /projects/:id/merge_requests "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -1282,6 +1299,7 @@ Must include at least one non-required attribute from above. "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -1467,6 +1485,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -1655,6 +1674,7 @@ Parameters: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -1956,6 +1976,7 @@ Example response: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -2115,6 +2136,7 @@ Example response: "Community contribution", "Manage" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 5, @@ -2291,6 +2313,7 @@ Example response: "source_project_id": 3, "target_project_id": 3, "labels": [], + "draft": false, "work_in_progress": false, "milestone": { "id": 27, diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index 9fc17930c92..f74f7785d30 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/notification_settings.md b/doc/api/notification_settings.md index 298c0ead8c1..69bed193f07 100644 --- a/doc/api/notification_settings.md +++ b/doc/api/notification_settings.md @@ -123,7 +123,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The group/project ID or path | +| `id` | integer or string | yes | The ID, or [URL-encoded path, of the group or project](README.md#namespaced-path-encoding). | Example response: @@ -149,7 +149,7 @@ curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The group/project ID or path | +| `id` | integer or string | yes | The ID, or [URL-encoded path, of the group or project](README.md#namespaced-path-encoding) | | `level` | string | no | The global notification level | | `new_note` | boolean | no | Enable/disable this notification | | `new_issue` | boolean | no | Enable/disable this notification | diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 61eaf0f36d7..f5c75aac0d9 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -284,7 +284,8 @@ HTTP Basic Authentication with the application's `client_id` and `client_secret` ```shell echo 'grant_type=password&username=&password=' > auth.txt -curl --data "@auth.txt" --user client_id:client_secret --request POST "https://gitlab.example.com/oauth/token" +curl --data "@auth.txt" --user client_id:client_secret \ + --request POST "https://gitlab.example.com/oauth/token" ``` Then, you receive a response containing the access token: diff --git a/doc/api/packages/composer.md b/doc/api/packages/composer.md index ebf3ffba92f..4f8e0a23c9c 100644 --- a/doc/api/packages/composer.md +++ b/doc/api/packages/composer.md @@ -71,7 +71,7 @@ Example response: ## V1 packages list -Given the V1 provider sha, returns a list of packages within the repository. Using Composer V2 is +Given the V1 provider SHA, returns a list of packages in the repository. Using Composer V2 is recommended over V1. ```plaintext @@ -81,7 +81,7 @@ GET group/:id/-/packages/composer/p/:sha | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | string | yes | The ID or full path of the group. | -| `sha` | string | yes | The provider sha, provided by the Composer [base request](#base-repository-request). | +| `sha` | string | yes | The provider SHA, provided by the Composer [base request](#base-repository-request). | ```shell curl --user : "https://gitlab.example.com/api/v4/group/1/-/packages/composer/p/082df4a5035f8725a12i4a3d2da5e6aaa966d06843d0a5c6d499313810427bd6" @@ -115,7 +115,7 @@ the symbol `%24` (see example below). | -------------- | ------ | -------- | ----------- | | `id` | string | yes | The ID or full path of the group. | | `package_name` | string | yes | The name of the package. | -| `sha` | string | yes | The sha digest of the package, provided by the [V1 packages list](#v1-packages-list). | +| `sha` | string | yes | The SHA digest of the package, provided by the [V1 packages list](#v1-packages-list). | ```shell curl --user : "https://gitlab.example.com/api/v4/group/1/-/packages/composer/my-org/my-composer-package%245c873497cdaa82eda35af5de24b789be92dfb6510baf117c42f03899c166b6e7" @@ -247,7 +247,8 @@ POST projects/:id/packages/composer | `branch` | string | no | The name of the branch to target for the package. | ```shell -curl --request POST --user : --data tag=v1.0.0 "https://gitlab.example.com/api/v4/projects/1/packages/composer" +curl --request POST --user : \ + --data tag=v1.0.0 "https://gitlab.example.com/api/v4/projects/1/packages/composer" ``` Example response: @@ -272,7 +273,7 @@ GET projects/:id/packages/composer/archives/:package_name | -------------- | ------ | -------- | ----------- | | `id` | string | yes | The ID or full path of the group. | | `package_name` | string | yes | The name of the package. | -| `sha` | string | yes | The target sha of the requested package version. | +| `sha` | string | yes | The target SHA of the requested package version. | ```shell curl --user : "https://gitlab.example.com/api/v4/projects/1/packages/composer/archives/my-org/my-composer-package.zip?sha=673594f85a55fe3c0eb45df7bd2fa9d95a1601ab" diff --git a/doc/api/packages/nuget.md b/doc/api/packages/nuget.md index ed61704770b..bbcb2cb9bc4 100644 --- a/doc/api/packages/nuget.md +++ b/doc/api/packages/nuget.md @@ -82,7 +82,7 @@ This writes the downloaded file to `MyNuGetPkg.1.3.0.17.nupkg` in the current di > Introduced in GitLab 12.8. -Download a NuGet package file: +Upload a NuGet package file: ```plaintext PUT projects/:id/packages/nuget diff --git a/doc/api/packages/pypi.md b/doc/api/packages/pypi.md index 531193e59e2..77ba028c447 100644 --- a/doc/api/packages/pypi.md +++ b/doc/api/packages/pypi.md @@ -20,11 +20,82 @@ These endpoints do not adhere to the standard API authentication methods. See the [PyPI package registry documentation](../../user/packages/pypi_repository/index.md) for details on which headers and token types are supported. -## Download a package file +## Download a package file from a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12. + +Download a PyPI package file. The [simple API](#group-level-simple-api-entry-point) +normally supplies this URL. + +```plaintext +GET groups/:id/packages/pypi/files/:sha256/:file_identifier +``` + +| Attribute | Type | Required | Description | +| ----------------- | ------ | -------- | ----------- | +| `id` | string | yes | The ID or full path of the group. | +| `sha256` | string | yes | The PyPI package file's sha256 checksum. | +| `file_identifier` | string | yes | The PyPI package file's name. | + +```shell +curl --user : "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" +``` + +To write the output to a file: + +```shell +curl --user : "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz +``` + +This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. + +## Group level simple API entry point + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12. + +Returns the package descriptor as an HTML file: + +```plaintext +GET groups/:id/packages/pypi/simple/:package_name +``` + +| Attribute | Type | Required | Description | +| -------------- | ------ | -------- | ----------- | +| `id` | string | yes | The ID or full path of the group. | +| `package_name` | string | yes | The name of the package. | + +```shell +curl --user : "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" +``` + +Example response: + +```html + + + + Links for my.pypi.package + + +

Links for my.pypi.package

+ my.pypi.package-0.0.1-py3-none-any.whl
my.pypi.package-0.0.1.tar.gz
+ + +``` + +To write the output to a file: + +```shell +curl --user : "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html +``` + +This writes the downloaded file to `simple.html` in the current directory. + +## Download a package file from a project > Introduced in GitLab 12.10. -Download a PyPI package file. The [simple API](#simple-api-entry-point) +Download a PyPI package file. The [simple API](#project-level-simple-api-entry-point) normally supplies this URL. ```plaintext @@ -49,7 +120,7 @@ curl --user : "https://gitlab.example.com/api/v This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. -## Simple API entry point +## Project-level simple API entry point > Introduced in GitLab 12.10. diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index f9bb8521a1f..fbea365e3d5 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -134,19 +134,24 @@ POST /projects/:id/pages/domains Create a new Pages domain with a certificate from a `.pem` file: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" \ + --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` Create a new Pages domain by using a variable containing the certificate: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" \ + --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` Create a new Pages domain with an [automatic certificate](../user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md#enabling-lets-encrypt-integration-for-your-custom-domain): ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "domain=ssl.domain.example" --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains" +curl --request POST --header "PRIVATE-TOKEN: " --form "domain=ssl.domain.example" \ + --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains" ``` ```json @@ -184,13 +189,15 @@ PUT /projects/:id/pages/domains/:domain Add a certificate for a Pages domain from a `.pem` file: ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" +curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=@/path/to/cert.pem" \ + --form "key=@/path/to/key.pem" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` Add a certificate for a Pages domain by using a variable containing the certificate: ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" +curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=$CERT_PEM" \ + --form "key=$KEY_PEM" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -210,7 +217,8 @@ curl --request PUT --header "PRIVATE-TOKEN: " --form "certifi ### Enabling Let's Encrypt integration for Pages custom domains ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form "auto_ssl_enabled=true" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json @@ -226,7 +234,8 @@ curl --request PUT --header "PRIVATE-TOKEN: " --form "auto_ss To remove the SSL certificate attached to the Pages domain, run: ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=" --form "key=" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" +curl --request PUT --header "PRIVATE-TOKEN: " --form "certificate=" \ + --form "key=" "https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example" ``` ```json diff --git a/doc/api/pipeline_schedules.md b/doc/api/pipeline_schedules.md index 6b3b6f4f36b..afb5d434fe7 100644 --- a/doc/api/pipeline_schedules.md +++ b/doc/api/pipeline_schedules.md @@ -1,10 +1,10 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Pipeline schedules API +# Pipeline schedules API **(FREE)** You can read more about [pipeline schedules](../ci/pipelines/schedules.md). @@ -30,7 +30,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a { "id": 13, "description": "Test schedule pipeline", - "ref": "master", + "ref": "main", "cron": "* * * * *", "cron_timezone": "Asia/Tokyo", "next_run_at": "2017-05-19T13:41:00.000Z", @@ -70,7 +70,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a { "id": 13, "description": "Test schedule pipeline", - "ref": "master", + "ref": "main", "cron": "* * * * *", "cron_timezone": "Asia/Tokyo", "next_run_at": "2017-05-19T13:41:00.000Z", @@ -80,7 +80,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a "last_pipeline": { "id": 332, "sha": "0e788619d0b5ec17388dffb973ecd505946156db", - "ref": "master", + "ref": "main", "status": "pending" }, "owner": { @@ -119,14 +119,16 @@ POST /projects/:id/pipeline_schedules | `active` | boolean | no | The activation of pipeline schedule. If false is set, the pipeline schedule is initially deactivated (default: `true`). | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form description="Build packages" --form ref="master" --form cron="0 1 * * 5" --form cron_timezone="UTC" --form active="true" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form description="Build packages" --form ref="main" --form cron="0 1 * * 5" --form cron_timezone="UTC" \ + --form active="true" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules" ``` ```json { "id": 14, "description": "Build packages", - "ref": "master", + "ref": "main", "cron": "0 1 * * 5", "cron_timezone": "UTC", "next_run_at": "2017-05-26T01:00:00.000Z", @@ -164,14 +166,15 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id | `active` | boolean | no | The activation of pipeline schedule. If false is set, the pipeline schedule is initially deactivated. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form cron="0 2 * * *" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form cron="0 2 * * *" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13" ``` ```json { "id": 13, "description": "Test schedule pipeline", - "ref": "master", + "ref": "main", "cron": "0 2 * * *", "cron_timezone": "Asia/Tokyo", "next_run_at": "2017-05-19T17:00:00.000Z", @@ -181,7 +184,7 @@ curl --request PUT --header "PRIVATE-TOKEN: " --form cron="0 "last_pipeline": { "id": 332, "sha": "0e788619d0b5ec17388dffb973ecd505946156db", - "ref": "master", + "ref": "main", "status": "pending" }, "owner": { @@ -216,7 +219,7 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla { "id": 13, "description": "Test schedule pipeline", - "ref": "master", + "ref": "main", "cron": "0 2 * * *", "cron_timezone": "Asia/Tokyo", "next_run_at": "2017-05-19T17:00:00.000Z", @@ -226,7 +229,7 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla "last_pipeline": { "id": 332, "sha": "0e788619d0b5ec17388dffb973ecd505946156db", - "ref": "master", + "ref": "main", "status": "pending" }, "owner": { @@ -261,7 +264,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: " "https://git { "id": 13, "description": "Test schedule pipeline", - "ref": "master", + "ref": "main", "cron": "0 2 * * *", "cron_timezone": "Asia/Tokyo", "next_run_at": "2017-05-19T17:00:00.000Z", @@ -271,7 +274,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: " "https://git "last_pipeline": { "id": 332, "sha": "0e788619d0b5ec17388dffb973ecd505946156db", - "ref": "master", + "ref": "main", "status": "pending" }, "owner": { @@ -317,8 +320,6 @@ Example response: ## Pipeline schedule variables -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34518) in GitLab 10.0. - ## Create a new pipeline schedule variable Create a new variable of a pipeline schedule. @@ -336,7 +337,8 @@ POST /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "key=NEW_VARIABLE" --form "value=new value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables" +curl --request POST --header "PRIVATE-TOKEN: " --form "key=NEW_VARIABLE" \ + --form "value=new value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables" ``` ```json @@ -364,7 +366,9 @@ PUT /projects/:id/pipeline_schedules/:pipeline_schedule_id/variables/:key | `variable_type` | string | no | The type of a variable. Available types are: `env_var` (default) and `file` | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "value=updated value" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form "value=updated value" \ + "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/variables/NEW_VARIABLE" ``` ```json diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index ea10b14bc2e..94122a40b2d 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -1,10 +1,10 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Pipeline triggers API +# Pipeline triggers API **(FREE)** You can read more about [triggering pipelines through the API](../ci/triggers/README.md). @@ -81,7 +81,8 @@ POST /projects/:id/triggers | `description` | string | yes | The trigger name | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form description="my description" "https://gitlab.example.com/api/v4/projects/1/triggers" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form description="my description" "https://gitlab.example.com/api/v4/projects/1/triggers" ``` ```json @@ -111,7 +112,8 @@ PUT /projects/:id/triggers/:trigger_id | `description` | string | no | The trigger name | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form description="my description" "https://gitlab.example.com/api/v4/projects/1/triggers/10" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form description="my description" "https://gitlab.example.com/api/v4/projects/1/triggers/10" ``` ```json diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 497c70b19ba..57c356ccf29 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -1,10 +1,10 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Pipelines API +# Pipelines API **(FREE)** ## Single Pipeline Requests @@ -23,8 +23,6 @@ Read more on [pagination](README.md#pagination). ## List project pipelines -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 - ```plaintext GET /projects/:id/pipelines ``` @@ -77,8 +75,6 @@ Example of response ## Get a single pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 - ```plaintext GET /projects/:id/pipelines/:pipeline_id ``` @@ -99,7 +95,7 @@ Example of response "id": 46, "project_id": 1, "status": "success", - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "before_sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "tag": false, @@ -213,8 +209,6 @@ Sample response: ## Create a new pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7209) in GitLab 8.14 - ```plaintext POST /projects/:id/pipeline ``` @@ -226,7 +220,7 @@ POST /projects/:id/pipeline | `variables` | array | no | An array containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }]` | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=master" +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=main" ``` Example of response @@ -236,7 +230,7 @@ Example of response "id": 61, "project_id": 1, "sha": "384c444e840a515b23f21915ee5766b87068a70d", - "ref": "master", + "ref": "main", "status": "pending", "before_sha": "0000000000000000000000000000000000000000", "tag": false, @@ -263,8 +257,6 @@ Example of response ## Retry jobs in a pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 - ```plaintext POST /projects/:id/pipelines/:pipeline_id/retry ``` @@ -285,7 +277,7 @@ Response: "id": 46, "project_id": 1, "status": "pending", - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "before_sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "tag": false, @@ -312,8 +304,6 @@ Response: ## Cancel a pipeline's jobs -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 - ```plaintext POST /projects/:id/pipelines/:pipeline_id/cancel ``` @@ -334,7 +324,7 @@ Response: "id": 46, "project_id": 1, "status": "canceled", - "ref": "master", + "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "before_sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", "tag": false, diff --git a/doc/api/project_aliases.md b/doc/api/project_aliases.md index 439f34ad93b..1638bb644c2 100644 --- a/doc/api/project_aliases.md +++ b/doc/api/project_aliases.md @@ -68,8 +68,8 @@ Example response: ## Create a project alias -Add a new alias for a project. Responds with a 201 when successful, -400 when there are validation errors (e.g. alias already exists): +Add a new alias for a project. When successful, responds with `201 Created`. +When there are validation errors, for example, when the alias already exists, responds with `400 Bad Request`: ```plaintext POST /project_aliases @@ -81,13 +81,15 @@ POST /project_aliases | `name` | string | yes | The name of the alias. Must be unique. | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/project_aliases" --form "project_id=1" --form "name=gitlab" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/project_aliases" --form "project_id=1" --form "name=gitlab" ``` or ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/project_aliases" --form "project_id=gitlab-org/gitlab" --form "name=gitlab" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/project_aliases" --form "project_id=gitlab-org/gitlab" --form "name=gitlab" ``` Example response: diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index a17f7d15e76..c6bcaa1ae9c 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -109,7 +109,9 @@ POST /projects/:id/badges | `name` | string | no | Name of the badge | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&name=mybadge" "https://gitlab.example.com/api/v4/projects/:id/badges" +curl --request POST --header "PRIVATE-TOKEN: " \ + --data "link_url=https://gitlab.com/gitlab-org/gitlab-foss/commits/master&image_url=https://shields.io/my/badge1&name=mybadge" \ + "https://gitlab.example.com/api/v4/projects/:id/badges" ``` Example response: diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index a431e754774..f31b3ccd0bb 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23922) in GitLab 11.7. -Users need at least [Maintainer](../user/permissions.md) access to use these endpoints. +Users need at least the [Maintainer](../user/permissions.md) role to use these endpoints. ## List project clusters @@ -22,7 +22,7 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ------- | -------- | ----------------------------------------------------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | Example request: @@ -92,7 +92,7 @@ Parameters: | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ----------------------------------------------------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `cluster_id` | integer | yes | The ID of the cluster | Example request: @@ -151,7 +151,8 @@ Example response: "path_with_namespace":"root/project-with-clusters-api", "created_at":"2019-01-02T20:13:32.600Z", "default_branch":null, - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo":"ssh://gitlab.example.com/root/project-with-clusters-api.git", "http_url_to_repo":"https://gitlab.example.com/root/project-with-clusters-api.git", "web_url":"https://gitlab.example.com/root/project-with-clusters-api", @@ -185,7 +186,7 @@ Parameters: | Attribute | Type | Required | Description | | ---------------------------------------------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | yes | The name of the cluster | | `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | | `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | @@ -247,7 +248,8 @@ Example response: "path_with_namespace":"root/project-with-clusters-api", "created_at":"2019-01-02T20:13:32.600Z", "default_branch":null, - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo":"ssh:://gitlab.example.com/root/project-with-clusters-api.git", "http_url_to_repo":"https://gitlab.example.com/root/project-with-clusters-api.git", "web_url":"https://gitlab.example.com/root/project-with-clusters-api", @@ -281,7 +283,7 @@ Parameters: | Attribute | Type | Required | Description | | ------------------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------ | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `cluster_id` | integer | yes | The ID of the cluster | | `name` | string | no | The name of the cluster | | `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | @@ -357,7 +359,8 @@ Example response: "path_with_namespace":"root/project-with-clusters-api", "created_at":"2019-01-02T20:13:32.600Z", "default_branch":null, - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo":"ssh:://gitlab.example.com/root/project-with-clusters-api.git", "http_url_to_repo":"https://gitlab.example.com/root/project-with-clusters-api.git", "web_url":"https://gitlab.example.com/root/project-with-clusters-api", @@ -392,7 +395,7 @@ Parameters: | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ----------------------------------------------------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `cluster_id` | integer | yes | The ID of the cluster | Example request: diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index a4ad496b667..b20ce9896dc 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -18,7 +18,7 @@ See also: Start a new export. -The endpoint also accepts an `upload` parameter. This parameter is a hash that contains +The endpoint also accepts an `upload` parameter. This parameter is a hash. It contains all the necessary information to upload the exported project to a web server or to any S3-compatible platform. At the moment we only support binary data file uploads to the final server. @@ -70,23 +70,14 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a Status can be one of: -- `none` -- `queued` -- `started` -- `finished` -- `regeneration_in_progress` - -`queued` state represents the request for export is received, and is currently in the queue to be processed. - -The `started` state represents that the export process has started and is currently in progress. -It includes the process of exporting, actions performed on the resultant file such as sending -an email notifying the user to download the file, uploading the exported file to a web server, etc. - -`finished` state is after the export process has completed and the user has been notified. - -`regeneration_in_progress` is when an export file is available to download, and a request to generate a new export is in process. - -`none` is when there are no exports _queued_, _started_, _finished_, or _being regenerated_ +- `none`: No exports _queued_, _started_, _finished_, or _being regenerated_. +- `queued`: The request for export is received, and is in the queue to be processed. +- `started`: The export process has started and is in progress. It includes: + - The process of exporting. + - Actions performed on the resulting file, such as sending an email notifying + the user to download the file, or uploading the exported file to a web server. +- `finished`: After the export process has completed and the user has been notified. +- `regeneration_in_progress`: An export file is available to download, and a request to generate a new export is in process. `_links` are only present when export has finished. @@ -122,7 +113,8 @@ GET /projects/:id/export/download | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | ```shell -curl --header "PRIVATE-TOKEN: " --remote-header-name --remote-name "https://gitlab.example.com/api/v4/projects/5/export/download" +curl --header "PRIVATE-TOKEN: " --remote-header-name \ + --remote-name "https://gitlab.example.com/api/v4/projects/5/export/download" ``` ```shell @@ -153,7 +145,8 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "path=api-project" --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/projects/import" +curl --request POST --header "PRIVATE-TOKEN: " --form "path=api-project" \ + --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/projects/import" ``` cURL doesn't support posting a file from a remote server. Importing a project from a remote server can be accomplished through something like the following: @@ -288,7 +281,7 @@ NOTE: An element's `id` field in `failed_relations` references the failure record, not the relation. NOTE: -The `failed_relations` array is currently capped to 100 items. +The `failed_relations` array is capped to 100 items. ```json { diff --git a/doc/api/project_level_variables.md b/doc/api/project_level_variables.md index 8ef887675e9..0b7193ad5bc 100644 --- a/doc/api/project_level_variables.md +++ b/doc/api/project_level_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference, api --- @@ -85,7 +85,8 @@ POST /projects/:id/variables | `environment_scope` | string | no | The `environment_scope` of the variable. Default: `*` | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/variables" --form "key=NEW_VARIABLE" --form "value=new value" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/variables" --form "key=NEW_VARIABLE" --form "value=new value" ``` ```json @@ -119,7 +120,8 @@ PUT /projects/:id/variables/:key | `filter` | hash | no | Available filters: `[environment_scope]`. See the [`filter` parameter details](#the-filter-parameter). | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/variables/NEW_VARIABLE" --form "value=updated value" +curl --request PUT --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/variables/NEW_VARIABLE" --form "value=updated value" ``` ```json diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index dd8954f2f0f..31c306bb14f 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -221,7 +221,8 @@ Example request: ```shell curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"destination_storage_name":"storage2"}' "https://gitlab.example.com/api/v4/projects/1/repository_storage_moves" + --data '{"destination_storage_name":"storage2"}' \ + "https://gitlab.example.com/api/v4/projects/1/repository_storage_moves" ``` Example response: @@ -266,7 +267,8 @@ Example request: ```shell curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"source_storage_name":"default"}' "https://gitlab.example.com/api/v4/project_repository_storage_moves" + --data '{"source_storage_name":"default"}' \ + "https://gitlab.example.com/api/v4/project_repository_storage_moves" ``` Example response: diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index 070429eafd5..156d3c57a43 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -16,7 +16,7 @@ Constants for snippet visibility levels are: | visibility | Description | | ---------- | ----------- | -| `private` | The snippet is visible only the snippet creator | +| `private` | The snippet is visible only to the snippet creator | | `internal` | The snippet is visible for any logged in user except [external users](../user/permissions.md#external-users) | | `public` | The snippet can be accessed without any authentication | @@ -225,7 +225,7 @@ Parameters: - `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user - `snippet_id` (required) - The ID of a project's snippet -- `ref` (required) - The name of a branch, tag or commit, such as `master` +- `ref` (required) - The name of a branch, tag or commit, such as `main` - `file_path` (required) - The URL-encoded path to the file, such as `snippet%2Erb` Example request: @@ -239,7 +239,7 @@ curl "https://gitlab.com/api/v4/projects/1/snippets/2/files/master/snippet%2Erb/ > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29508) in GitLab 9.4. -Available only for users with Administrator [permissions](../user/permissions.md). +Available only for users with the Administrator [role](../user/permissions.md). ```plaintext GET /projects/:id/snippets/:snippet_id/user_agent_detail @@ -247,7 +247,7 @@ GET /projects/:id/snippets/:snippet_id/user_agent_detail | Attribute | Type | Required | Description | |---------------|---------|----------|--------------------------------------| -| `id` | Integer | yes | The ID of a project | +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](README.md#namespaced-path-encoding). | | `snippet_id` | Integer | yes | The ID of a snippet | Example request: diff --git a/doc/api/projects.md b/doc/api/projects.md index b686d17a4a1..e5cb2c8e1eb 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -59,7 +59,7 @@ GET /projects | `sort` | string | **{dotted-circle}** No | Return projects sorted in `asc` or `desc` order. Default is `desc`. | | `starred` | boolean | **{dotted-circle}** No | Limit by projects starred by the current user. | | `statistics` | boolean | **{dotted-circle}** No | Include project statistics. | -| `topic` | string | **{dotted-circle}** No | Comma-separated topic names. Limit results to projects that match all of given topics. See `tag_list` attribute. | +| `topic` | string | **{dotted-circle}** No | Comma-separated topic names. Limit results to projects that match all of given topics. See `topics` attribute. | | `visibility` | string | **{dotted-circle}** No | Limit by visibility `public`, `internal`, or `private`. | | `wiki_checksum_failed` **(PREMIUM)** | boolean | **{dotted-circle}** No | Limit projects where the wiki checksum calculation has failed ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6137) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2). | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(admins only)_ | @@ -82,7 +82,11 @@ When `simple=true` or the user is unauthenticated this returns something like: "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", "web_url": "http://example.com/diaspora/diaspora-client", "readme_url": "http://example.com/diaspora/diaspora-client/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora client" + ], + "topics": [ "example", "disapora client" ], @@ -116,7 +120,11 @@ When the user is authenticated and `simple` is not set this returns something li "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", "web_url": "http://example.com/diaspora/diaspora-client", "readme_url": "http://example.com/diaspora/diaspora-client/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora client" + ], + "topics": [ "example", "disapora client" ], @@ -166,6 +174,7 @@ When the user is authenticated and `simple` is not set this returns something li "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on @@ -200,7 +209,11 @@ When the user is authenticated and `simple` is not set this returns something li "http_url_to_repo": "http://example.com/brightbox/puppet.git", "web_url": "http://example.com/brightbox/puppet", "readme_url": "http://example.com/brightbox/puppet/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "puppet" + ], + "topics": [ "example", "puppet" ], @@ -261,6 +274,7 @@ When the user is authenticated and `simple` is not set this returns something li "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "repository_storage": "default", @@ -300,6 +314,10 @@ When the user is authenticated and `simple` is not set this returns something li ] ``` +NOTE: +The `tag_list` attribute has been deprecated +and is removed in API v5 in favor of the `topics` attribute. + NOTE: For users of [GitLab Premium or higher](https://about.gitlab.com/pricing/), the `marked_for_deletion_at` attribute has been deprecated, and is removed @@ -378,7 +396,11 @@ GET /users/:user_id/projects "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", "web_url": "http://example.com/diaspora/diaspora-client", "readme_url": "http://example.com/diaspora/diaspora-client/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora client" + ], + "topics": [ "example", "disapora client" ], @@ -428,6 +450,7 @@ GET /users/:user_id/projects "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "marked_for_deletion_at": "2020-04-03", // Deprecated and will be removed in API v5 in favor of marked_for_deletion_on @@ -462,7 +485,11 @@ GET /users/:user_id/projects "http_url_to_repo": "http://example.com/brightbox/puppet.git", "web_url": "http://example.com/brightbox/puppet", "readme_url": "http://example.com/brightbox/puppet/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "puppet" + ], + "topics": [ "example", "puppet" ], @@ -523,6 +550,7 @@ GET /users/:user_id/projects "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "repository_storage": "default", @@ -606,7 +634,11 @@ Example response: "http_url_to_repo": "http://example.com/diaspora/diaspora-client.git", "web_url": "http://example.com/diaspora/diaspora-client", "readme_url": "http://example.com/diaspora/diaspora-client/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora client" + ], + "topics": [ "example", "disapora client" ], @@ -654,6 +686,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "statistics": { @@ -683,7 +716,11 @@ Example response: "http_url_to_repo": "http://example.com/brightbox/puppet.git", "web_url": "http://example.com/brightbox/puppet", "readme_url": "http://example.com/brightbox/puppet/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "puppet" + ], + "topics": [ "example", "puppet" ], @@ -742,6 +779,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "repository_storage": "default", @@ -789,7 +827,7 @@ GET /projects/:id | Attribute | Type | Required | Description | |--------------------------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `license` | boolean | **{dotted-circle}** No | Include project license data. | | `statistics` | boolean | **{dotted-circle}** No | Include project statistics. | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(admins only)_ | @@ -804,7 +842,11 @@ GET /projects/:id "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -900,6 +942,7 @@ GET /projects/:id "printing_merge_requests_link_enabled": true, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "approvals_before_merge": 0, @@ -940,6 +983,10 @@ GET /projects/:id } ``` +NOTE: +The `tag_list` attribute has been deprecated +and is removed in API v5 in favor of the `topics` attribute. + Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) can also see the `approvals_before_merge` parameter: @@ -974,12 +1021,13 @@ If the project is a fork, and you provide a valid token to authenticate, the "path_with_namespace":"gitlab-org/gitlab-foss", "created_at":"2013-09-26T06:02:36.000Z", "default_branch":"master", - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo":"git@gitlab.com:gitlab-org/gitlab-foss.git", "http_url_to_repo":"https://gitlab.com/gitlab-org/gitlab-foss.git", "web_url":"https://gitlab.com/gitlab-org/gitlab-foss", "avatar_url":"https://assets.gitlab-static.net/uploads/-/system/project/avatar/13083/logo-extra-whitespace.png", - "license_url": "https://gitlab.com/gitlab-org/gitlab/blob/master/LICENSE", + "license_url": "https://gitlab.com/gitlab-org/gitlab/-/blob/master/LICENSE", "license": { "key": "mit", "name": "MIT License", @@ -1032,7 +1080,7 @@ GET /projects/:id/users | Attribute | Type | Required | Description | |--------------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `search` | string | **{dotted-circle}** No | Search for specific users. | | `skip_users` | integer array | **{dotted-circle}** No | Filter out users with the specified IDs. | @@ -1067,7 +1115,7 @@ GET /projects/:id/groups | Attribute | Type | Required | Description | |-----------------------------|-------------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `search` | string | **{dotted-circle}** No | Search for specific groups. | | `skip_groups` | array of integers | **{dotted-circle}** No | Skip the group IDs passed. | | `with_shared` | boolean | **{dotted-circle}** No | Include projects shared with this group. Default is `false`. | @@ -1117,7 +1165,7 @@ POST /projects | `path` | string | **{check-circle}** Yes (if name isn't provided) | Repository name for new project. Generated based on name if not provided (generated as lowercase with dashes). | | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | | `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | @@ -1130,7 +1178,7 @@ POST /projects | `ci_config_path` | string | **{dotted-circle}** No | The path to CI configuration file. | | `container_expiration_policy_attributes` | hash | **{dotted-circle}** No | Update the image cleanup policy for this project. Accepts: `cadence` (string), `keep_n` (integer), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean). Valid values for `cadence` are: `1d` (every day), `7d` (every week), `14d` (every two weeks), `1month` (every month), or `3month` (every quarter). | | `container_registry_enabled` | boolean | **{dotted-circle}** No | Enable container registry for this project. | -| `default_branch` | string | **{dotted-circle}** No | The [default branch](../user/project/repository/branches/default.md) name. | +| `default_branch` | string | **{dotted-circle}** No | The [default branch](../user/project/repository/branches/default.md) name. Requires `initialize_with_readme` to be `true`. | | `description` | string | **{dotted-circle}** No | Short project description. | | `emails_disabled` | boolean | **{dotted-circle}** No | Disable email notifications. | | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | @@ -1165,9 +1213,11 @@ POST /projects | `show_default_award_emojis` | boolean | **{dotted-circle}** No | Show default award emojis. | | `snippets_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | -| `tag_list` | array | **{dotted-circle}** No | The list of tags for a project; put array of tags, that should be finally assigned to a project. | +| `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | +| `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | | `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/working_with_projects.md#built-in-templates). When used with `use_custom_template`, name of a custom project template. | | `template_project_id` **(PREMIUM)** | integer | **{dotted-circle}** No | When used with `use_custom_template`, project ID of a custom project template. This is preferable to using `template_name` since `template_name` may be ambiguous. | +| `topics` | array | **{dotted-circle}** No | The list of topics for a project; put array of topics, that should be finally assigned to a project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `use_custom_template` **(PREMIUM)** | boolean | **{dotted-circle}** No | Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template. | | `visibility` | string | **{dotted-circle}** No | See [project visibility level](#project-visibility-level). | | `wiki_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1191,7 +1241,7 @@ POST /projects/user/:user_id | `name` | string | **{check-circle}** Yes | The name of the new project. | | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | | `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | @@ -1204,6 +1254,7 @@ POST /projects/user/:user_id | `ci_config_path` | string | **{dotted-circle}** No | The path to CI configuration file. | | `container_registry_enabled` | boolean | **{dotted-circle}** No | Enable container registry for this project. | | `description` | string | **{dotted-circle}** No | Short project description. | +| `default_branch` | string | **{dotted-circle}** No | The [default branch](../user/project/repository/branches/default.md) name. Requires `initialize_with_readme` to be `true`. | | `emails_disabled` | boolean | **{dotted-circle}** No | Disable email notifications. | | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | | `forking_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1238,9 +1289,11 @@ POST /projects/user/:user_id | `show_default_award_emojis` | boolean | **{dotted-circle}** No | Show default award emojis. | | `snippets_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | -| `suggestion_commit_message` | string | **{dotted-circle}** No | The commit message used to apply merge request suggestions. | -| `tag_list` | array | **{dotted-circle}** No | The list of tags for a project; put array of tags, that should be finally assigned to a project. | +| `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | +| `suggestion_commit_message` | string | **{dotted-circle}** No | The commit message used to apply merge request [suggestions](../user/project/merge_requests/reviews/suggestions.md). | +| `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | | `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/working_with_projects.md#built-in-templates). When used with `use_custom_template`, name of a custom project template. | +| `topics` | array | **{dotted-circle}** No | The list of topics for the project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `use_custom_template` **(PREMIUM)** | boolean | **{dotted-circle}** No | Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template. | | `visibility` | string | **{dotted-circle}** No | See [project visibility level](#project-visibility-level). | | `wiki_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1262,7 +1315,7 @@ PUT /projects/:id |-------------------------------------------------------------|----------------|------------------------|-------------| | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge request by default. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge request by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | | `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual`, or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | @@ -1282,7 +1335,7 @@ PUT /projects/:id | `emails_disabled` | boolean | **{dotted-circle}** No | Disable email notifications. | | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | | `forking_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `import_url` | string | **{dotted-circle}** No | URL to import repository from. | | `issues_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `issues_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable issues for this project. Use `issues_access_level` instead. | @@ -1316,13 +1369,16 @@ PUT /projects/:id | `show_default_award_emojis` | boolean | **{dotted-circle}** No | Show default award emojis. | | `snippets_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | +| `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | | `suggestion_commit_message` | string | **{dotted-circle}** No | The commit message used to apply merge request suggestions. | -| `tag_list` | array | **{dotted-circle}** No | The list of tags for a project; put array of tags, that should be finally assigned to a project. | +| `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | +| `topics` | array | **{dotted-circle}** No | The list of topics for the project. This replaces any existing topics that are already added to the project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `visibility` | string | **{dotted-circle}** No | See [project visibility level](#project-visibility-level). | | `wiki_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `wiki_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable wiki for this project. Use `wiki_access_level` instead. | | `issues_template` **(PREMIUM)** | string | **{dotted-circle}** No | Default description for Issues. Description is parsed with GitLab Flavored Markdown. See [Templates for issues and merge requests](#templates-for-issues-and-merge-requests). | | `merge_requests_template` **(PREMIUM)** | string | **{dotted-circle}** No | Default description for Merge Requests. Description is parsed with GitLab Flavored Markdown. See [Templates for issues and merge requests](#templates-for-issues-and-merge-requests). | +| `keep_latest_artifact` | boolean | **{dotted-circle}** No | Disable or enable the ability to keep the latest artifact for this project. | ## Fork project @@ -1338,11 +1394,11 @@ POST /projects/:id/fork | Attribute | Type | Required | Description | |------------------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `name` | string | **{dotted-circle}** No | The name assigned to the resultant project after forking. | | `namespace_id` | integer | **{dotted-circle}** No | The ID of the namespace that the project is forked to. | | `namespace_path` | string | **{dotted-circle}** No | The path of the namespace that the project is forked to. | -| `namespace` | integer/string | **{dotted-circle}** No | _(Deprecated)_ The ID or path of the namespace that the project is forked to. | +| `namespace` | integer or string | **{dotted-circle}** No | _(Deprecated)_ The ID or path of the namespace that the project is forked to. | | `path` | string | **{dotted-circle}** No | The path assigned to the resultant project after forking. | | `description` | string | **{dotted-circle}** No | The description assigned to the resultant project after forking. | | `visibility` | string | **{dotted-circle}** No | The [visibility level](#project-visibility-level) assigned to the resultant project after forking. | @@ -1361,7 +1417,7 @@ GET /projects/:id/forks | Attribute | Type | Required | Description | |-------------------------------|----------------|------------------------|-------------| | `archived` | boolean | **{dotted-circle}** No | Limit by archived status. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `membership` | boolean | **{dotted-circle}** No | Limit by projects that the current user is a member of. | | `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [access level](members.md#valid-access-levels). | | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | @@ -1393,7 +1449,11 @@ Example responses: "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -1435,6 +1495,7 @@ Example responses: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-project-site", @@ -1462,7 +1523,7 @@ POST /projects/:id/star | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/star" @@ -1480,7 +1541,11 @@ Example response: "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -1530,6 +1595,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-project-site", @@ -1555,7 +1621,7 @@ POST /projects/:id/unstar | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/unstar" @@ -1573,7 +1639,11 @@ Example response: "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -1623,6 +1693,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-project-site", @@ -1648,7 +1719,7 @@ GET /projects/:id/starrers | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `search` | string | **{dotted-circle}** No | Search for specific users. | ```shell @@ -1694,7 +1765,7 @@ GET /projects/:id/languages | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/languages" @@ -1723,7 +1794,7 @@ POST /projects/:id/archive | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/archive" @@ -1741,7 +1812,11 @@ Example response: "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -1810,6 +1885,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-project-site", @@ -1837,7 +1913,7 @@ POST /projects/:id/unarchive | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/unarchive" @@ -1855,7 +1931,11 @@ Example response: "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", "web_url": "http://example.com/diaspora/diaspora-project-site", "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", - "tag_list": [ + "tag_list": [ //deprecated, use `topics` instead + "example", + "disapora project" + ], + "topics": [ "example", "disapora project" ], @@ -1924,6 +2004,7 @@ Example response: "remove_source_branch_after_merge": false, "request_access_enabled": false, "merge_method": "merge", + "squash_option": "default_on", "autoclose_referenced_issues": true, "suggestion_commit_message": null, "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-project-site", @@ -1963,7 +2044,7 @@ DELETE /projects/:id | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Restore project marked for deletion **(PREMIUM)** @@ -1977,12 +2058,13 @@ POST /projects/:id/restore | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Upload a file Uploads a file to the specified project to be used in an issue or merge request -description, or a comment. +description, or a comment. GitLab versions 14.0 and later +[enforce](#max-attachment-size-enforcement) this limit. ```plaintext POST /projects/:id/uploads @@ -1991,7 +2073,7 @@ POST /projects/:id/uploads | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| | `file` | string | **{check-circle}** Yes | The file to be uploaded. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | To upload a file from your file system, use the `--form` argument. This causes cURL to post data using the header `Content-Type: multipart/form-data`. The @@ -1999,7 +2081,8 @@ cURL to post data using the header `Content-Type: multipart/form-data`. The `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/5/uploads" ``` Returned object: @@ -2021,7 +2104,8 @@ the format in `markdown` is used. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57250) in GitLab 13.11. -GitLab 13.11 added enforcement of the [maximum attachment size limit](../user/admin_area/settings/account_and_limit_settings.md#max-attachment-size) behind the `enforce_max_attachment_size_upload_api` feature flag. GitLab 14.0 will enable this by default. +GitLab 13.11 added enforcement of the [maximum attachment size limit](../user/admin_area/settings/account_and_limit_settings.md#max-attachment-size) behind the `enforce_max_attachment_size_upload_api` feature flag. GitLab 14.0 enables this by default. +To disable this enforcement: **In Omnibus installations:** @@ -2031,10 +2115,10 @@ GitLab 13.11 added enforcement of the [maximum attachment size limit](../user/ad sudo gitlab-rails console ``` -1. Enable the feature flag: +1. Disable the feature flag: ```ruby - Feature.enable(:enforce_max_attachment_size_upload_api) + Feature.disable(:enforce_max_attachment_size_upload_api) ``` **In installations from source:** @@ -2046,10 +2130,10 @@ GitLab 13.11 added enforcement of the [maximum attachment size limit](../user/ad sudo -u git -H bundle exec rails console -e production ``` -1. Enable the feature flag to disable the validation: +1. Disable the feature flag: ```ruby - Feature.enable(:enforce_max_attachment_size_upload_api) + Feature.disable(:enforce_max_attachment_size_upload_api) ``` ## Upload a project avatar @@ -2063,7 +2147,7 @@ PUT /projects/:id | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| | `avatar` | string | **{check-circle}** Yes | The file to be uploaded. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | To upload an avatar from your file system, use the `--form` argument. This causes cURL to post data using the header `Content-Type: multipart/form-data`. The @@ -2073,7 +2157,8 @@ preceded by `@`. For example: Example request: ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "avatar=@dk.png" "https://gitlab.example.com/api/v4/projects/5" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form "avatar=@dk.png" "https://gitlab.example.com/api/v4/projects/5" ``` Returned object: @@ -2097,7 +2182,7 @@ POST /projects/:id/share | `expires_at` | string | **{dotted-circle}** No | Share expiration date in ISO 8601 format: 2016-09-26 | | `group_access` | integer | **{check-circle}** Yes | The [access level](members.md#valid-access-levels) to grant the group. | | `group_id` | integer | **{check-circle}** Yes | The ID of the group to share with. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Delete a shared project link within a group @@ -2110,7 +2195,7 @@ DELETE /projects/:id/share/:group_id | Attribute | Type | Required | Description | |------------|----------------|------------------------|-------------| | `group_id` | integer | **{check-circle}** Yes | The ID of the group. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/share/17" @@ -2131,7 +2216,7 @@ GET /projects/:id/hooks | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ### Get project hook @@ -2144,7 +2229,7 @@ GET /projects/:id/hooks/:hook_id | Attribute | Type | Required | Description | |-----------|----------------|------------------------|---------------------------| | `hook_id` | integer | **{check-circle}** Yes | The ID of a project hook. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```json { @@ -2183,7 +2268,7 @@ POST /projects/:id/hooks | `confidential_note_events` | boolean | **{dotted-circle}** No | Trigger hook on confidential note events. | | `deployment_events` | boolean | **{dotted-circle}** No | Trigger hook on deployment events. | | `enable_ssl_verification` | boolean | **{dotted-circle}** No | Do SSL verification when triggering the hook. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `issues_events` | boolean | **{dotted-circle}** No | Trigger hook on issues events. | | `job_events` | boolean | **{dotted-circle}** No | Trigger hook on job events. | | `merge_requests_events` | boolean | **{dotted-circle}** No | Trigger hook on merge requests events. | @@ -2211,7 +2296,7 @@ PUT /projects/:id/hooks/:hook_id | `deployment_events` | boolean | **{dotted-circle}** No | Trigger hook on deployment events. | | `enable_ssl_verification` | boolean | **{dotted-circle}** No | Do SSL verification when triggering the hook. | | `hook_id` | integer | **{check-circle}** Yes | The ID of the project hook. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `issues_events` | boolean | **{dotted-circle}** No | Trigger hook on issues events. | | `job_events` | boolean | **{dotted-circle}** No | Trigger hook on job events. | | `merge_requests_events` | boolean | **{dotted-circle}** No | Trigger hook on merge requests events. | @@ -2237,7 +2322,7 @@ DELETE /projects/:id/hooks/:hook_id | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| | `hook_id` | integer | **{check-circle}** Yes | The ID of the project hook. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | Note the JSON response differs if the hook is available or not. If the project hook is available before it's returned in the JSON response or an empty response @@ -2257,7 +2342,7 @@ POST /projects/:id/fork/:forked_from_id | Attribute | Type | Required | Description | |------------------|----------------|------------------------|-------------| | `forked_from_id` | ID | **{check-circle}** Yes | The ID of the project that was forked from. | -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ### Delete an existing forked from relationship @@ -2267,7 +2352,7 @@ DELETE /projects/:id/fork | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Search for projects by name @@ -2297,7 +2382,7 @@ POST /projects/:id/housekeeping | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID of the project or NAMESPACE/PROJECT_NAME. | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Push Rules **(PREMIUM)** @@ -2312,7 +2397,7 @@ GET /projects/:id/push_rule | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID of the project or NAMESPACE/PROJECT_NAME. | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```json { @@ -2364,7 +2449,7 @@ POST /projects/:id/push_rule | `commit_message_regex` | string | **{dotted-circle}** No | All commit messages must match this, for example `Fixed \d+\..*`. | | `deny_delete_tag` | boolean | **{dotted-circle}** No | Deny deleting a tag. | | `file_name_regex` | string | **{dotted-circle}** No | All committed filenames must **not** match this, for example `(jar|exe)$`. | -| `id` | integer/string | **{check-circle}** Yes | The ID of the project or NAMESPACE/PROJECT_NAME. | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding), | | `max_file_size` | integer | **{dotted-circle}** No | Maximum file size (MB). | | `member_check` | boolean | **{dotted-circle}** No | Restrict commits by author (email) to existing GitLab users. | | `prevent_secrets` | boolean | **{dotted-circle}** No | GitLab rejects any files that are likely to contain secrets. | @@ -2387,7 +2472,7 @@ PUT /projects/:id/push_rule | `commit_message_regex` | string | **{dotted-circle}** No | All commit messages must match this, for example `Fixed \d+\..*`. | | `deny_delete_tag` | boolean | **{dotted-circle}** No | Deny deleting a tag. | | `file_name_regex` | string | **{dotted-circle}** No | All committed filenames must **not** match this, for example `(jar|exe)$`. | -| `id` | integer/string | **{check-circle}** Yes | The ID of the project or NAMESPACE/PROJECT_NAME. | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `max_file_size` | integer | **{dotted-circle}** No | Maximum file size (MB). | | `member_check` | boolean | **{dotted-circle}** No | Restrict commits by author (email) to existing GitLab users. | | `prevent_secrets` | boolean | **{dotted-circle}** No | GitLab rejects any files that are likely to contain secrets. | @@ -2406,7 +2491,7 @@ DELETE /projects/:id/push_rule | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ## Transfer a project to a new namespace @@ -2418,8 +2503,8 @@ PUT /projects/:id/transfer | Attribute | Type | Required | Description | |-------------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | -| `namespace` | integer/string | **{check-circle}** Yes | The ID or path of the namespace to transfer to project to. | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `namespace` | integer or string | **{check-circle}** Yes | The ID or path of the namespace to transfer to project to. | Example request: @@ -2439,7 +2524,8 @@ Example response: "path_with_namespace": "cute-cats/hello-world", "created_at": "2020-10-15T16:25:22.415Z", "default_branch": "master", - "tag_list": [], + "tag_list": [], //deprecated, use `topics` instead + "topics": [], "ssh_url_to_repo": "git@gitlab.example.com:cute-cats/hello-world.git", "http_url_to_repo": "https://gitlab.example.com/cute-cats/hello-world.git", "web_url": "https://gitlab.example.com/cute-cats/hello-world", @@ -2521,6 +2607,7 @@ Example response: "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", + "squash_option": "default_on", "suggestion_commit_message": null, "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", @@ -2566,7 +2653,7 @@ POST /projects/:id/mirror/pull | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/:id/mirror/pull" @@ -2595,5 +2682,30 @@ GET /projects/:id/snapshot | Attribute | Type | Required | Description | |-----------|----------------|------------------------|-------------| -| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | | `wiki` | boolean | **{dotted-circle}** No | Whether to download the wiki, rather than project, repository. | + +## Get the path to repository storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29861) in GitLab 14.0. + +Get the path to repository storage for specified project. Available for administrators only. + +```plaintext +GET /projects/:id/storage +``` + +| Attribute | Type | Required | Description | +|--------------|----------------|------------------------|-------------| +| `id` | integer or string | **{check-circle}** Yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding). | + +```json +[ + { + "project_id": 1, + "disk_path": "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", + "created_at": "2012-10-12T17:04:47Z", + "repository_storage": "default" + } +] +``` diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index e5560360532..2fe821a7758 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -22,7 +22,7 @@ The access levels are defined in the `ProtectedRefAccess.allowed_access_levels` ## List protected branches -Gets a list of protected branches from a project. +Gets a list of protected branches from a project as they are defined [in the UI](../user/project/protected_branches.md#configure-a-protected-branch). If a wildcard is set, it is returned instead of the exact name of the branches that match that wildcard. ```plaintext GET /projects/:id/protected_branches @@ -59,6 +59,24 @@ Example response: "allow_force_push":false, "code_owner_approval_required": false }, + { + "id": 1, + "name": "release/*", + "push_access_levels": [ + { + "access_level": 40, + "access_level_description": "Maintainers" + } + ], + "merge_access_levels": [ + { + "access_level": 40, + "access_level_description": "Maintainers" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false + }, ... ] ``` @@ -183,10 +201,10 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | yes | The name of the branch or wildcard | -| `push_access_level` | string | no | Access levels allowed to push (defaults: `40`, maintainer access level) | -| `merge_access_level` | string | no | Access levels allowed to merge (defaults: `40`, maintainer access level) | -| `unprotect_access_level` | string | no | Access levels allowed to unprotect (defaults: `40`, maintainer access level) | -| `allow_force_push` | boolean | no | Allow force push for all users with push access. (defaults: false) | +| `push_access_level` | string | no | Access levels allowed to push (defaults: `40`, Maintainer role) | +| `merge_access_level` | string | no | Access levels allowed to merge (defaults: `40`, Maintainer role) | +| `unprotect_access_level` | string | no | Access levels allowed to unprotect (defaults: `40`, Maintainer role) | +| `allow_force_push` | boolean | no | Allow all users with push access to force push. (default: `false`) | | `allowed_to_push` | array | no | **(PREMIUM)** Array of access levels allowed to push, with each described by a hash | | `allowed_to_merge` | array | no | **(PREMIUM)** Array of access levels allowed to merge, with each described by a hash | | `allowed_to_unprotect` | array | no | **(PREMIUM)** Array of access levels allowed to unprotect, with each described by a hash | diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index 487537cd3f5..52fcdad1ad6 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -96,7 +96,10 @@ POST /projects/:id/protected_environments ``` ```shell -curl --header 'Content-Type: application/json' --request POST --data '{"name": "production", "deploy_access_levels": [{"group_id": 9899826}]}' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/22034114/protected_environments" +curl --header 'Content-Type: application/json' --request POST \ + --data '{"name": "production", "deploy_access_levels": [{"group_id": 9899826}]}' \ + --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/22034114/protected_environments" ``` | Attribute | Type | Required | Description | diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md index 63fa7183aab..e04f418258d 100644 --- a/doc/api/protected_tags.md +++ b/doc/api/protected_tags.md @@ -102,7 +102,7 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | yes | The name of the tag or wildcard | -| `create_access_level` | string | no | Access levels allowed to create (defaults: `40`, maintainer access level) | +| `create_access_level` | string | no | Access levels allowed to create (defaults: `40`, Maintainer role) | Example response: diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index bc37f1aa0a9..0cbf613c598 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -542,7 +542,8 @@ PUT /projects/:id/releases/:tag_name Example request: ```shell -curl --header 'Content-Type: application/json' --request PUT --data '{"name": "new name", "milestones": ["v1.2"]}' --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/24/releases/v0.1" +curl --header 'Content-Type: application/json' --request PUT --data '{"name": "new name", "milestones": ["v1.2"]}' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/24/releases/v0.1" ``` Example response: diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index 911aa8bbbd0..f4a2df5558f 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -148,7 +148,9 @@ You have to specify at least one of `name` or `url` Example request: ```shell -curl --request PUT --data name="new name" --data link_type="runbook" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" +curl --request PUT --data name="new name" --data link_type="runbook" \ + --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/24/releases/v0.1/assets/links/1" ``` Example response: diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index c85454d66ee..5e5b44fec8a 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -71,7 +71,8 @@ POST /projects/:id/remote_mirrors Example request: ```shell -curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/42/remote_mirrors" +curl --request POST --data "url=https://username:token@example.com/gitlab/example.git" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/42/remote_mirrors" ``` Example response: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 857cd3883c8..1868f33373c 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -311,6 +311,11 @@ Supported attributes: | `file` | string | no | The file to commit the changes to, defaults to `CHANGELOG.md`. | | `message` | string | no | The commit message to produce when committing the changes, defaults to `Add changelog for version X` where X is the value of the `version` argument. | +WARNING: +GitLab treats trailers case-sensitively. If you set the `trailer` field to +`Example`, GitLab _won't_ include commits that use the trailer `example`, +`eXaMpLE`, or anything else that isn't _exactly_ `Example`. + If the `from` attribute is unspecified, GitLab uses the Git tag of the last stable version that came before the version specified in the `version` attribute. This requires that Git tag names follow a specific format, allowing diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 70b804c368e..0dc50543f1e 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -183,10 +183,11 @@ POST /projects/:id/repository/files/:file_path ``` ```shell -curl --request POST --header 'PRIVATE-TOKEN: ' --header "Content-Type: application/json" \ - --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ - "content": "some content", "commit_message": "create a new file"}' \ - "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" +curl --request POST --header 'PRIVATE-TOKEN: ' \ + --header "Content-Type: application/json" \ + --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ + "content": "some content", "commit_message": "create a new file"}' \ + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Example response: @@ -218,10 +219,11 @@ PUT /projects/:id/repository/files/:file_path ``` ```shell -curl --request PUT --header 'PRIVATE-TOKEN: ' --header "Content-Type: application/json" \ - --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ - "content": "some content", "commit_message": "update file"}' \ - "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" +curl --request PUT --header 'PRIVATE-TOKEN: ' \ + --header "Content-Type: application/json" \ + --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ + "content": "some content", "commit_message": "update file"}' \ + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Example response: @@ -253,7 +255,7 @@ error message. Possible causes for a failed commit include: user tried to make an empty commit; - the branch was updated by a Git push while the file edit was in progress. -Currently GitLab Shell has a boolean return code, preventing GitLab from specifying the error. +GitLab Shell has a boolean return code, preventing GitLab from specifying the error. ## Delete existing file in repository @@ -264,10 +266,11 @@ DELETE /projects/:id/repository/files/:file_path ``` ```shell -curl --request DELETE --header 'PRIVATE-TOKEN: ' --header "Content-Type: application/json" \ - --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ - "commit_message": "delete file"}' \ - "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" +curl --request DELETE --header 'PRIVATE-TOKEN: ' \ + --header "Content-Type: application/json" \ + --data '{"branch": "master", "author_email": "author@example.com", "author_name": "Firstname Lastname", \ + "commit_message": "delete file"}' \ + "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fproject%2Erb" ``` Parameters: diff --git a/doc/api/resource_access_tokens.md b/doc/api/resource_access_tokens.md index ecc5b3bf172..3b443dbb8f8 100644 --- a/doc/api/resource_access_tokens.md +++ b/doc/api/resource_access_tokens.md @@ -10,7 +10,7 @@ You can read more about [project access tokens](../user/project/settings/project ## List project access tokens -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. Get a list of project access tokens. @@ -20,7 +20,7 @@ GET projects/:id/access_tokens | Attribute | Type | required | Description | |-----------|---------|----------|---------------------| -| `id` | integer/string | yes | The ID of the project | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects//access_tokens" @@ -45,7 +45,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a ## Create a project access token -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55408) in GitLab 13.10. Create a project access token. @@ -55,9 +55,9 @@ POST projects/:id/access_tokens | Attribute | Type | required | Description | |-----------|---------|----------|---------------------| -| `id` | integer/string | yes | The ID of the project | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | | `name` | String | yes | The name of the project access token | -| `scopes` | Array\[String] | yes | [List of scopes](../user/project/settings/project_access_tokens.md#limiting-scopes-of-a-project-access-token) | +| `scopes` | `Array[String]` | yes | [List of scopes](../user/project/settings/project_access_tokens.md#limiting-scopes-of-a-project-access-token) | | `expires_at` | Date | no | The token expires at midnight UTC on that date | ```shell @@ -86,7 +86,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ ## Revoke a project access token -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. Revoke a project access token. @@ -96,8 +96,8 @@ DELETE projects/:id/access_tokens/:token_id | Attribute | Type | required | Description | |-----------|---------|----------|---------------------| -| `id` | integer/string | yes | The ID of the project | -| `token_id` | integer/string | yes | The ID of the project access token | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | +| `token_id` | integer or string | yes | The ID of the project access token | ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects//access_tokens/" diff --git a/doc/api/resource_label_events.md b/doc/api/resource_label_events.md index 0c1735c0664..5fc7a0a52bd 100644 --- a/doc/api/resource_label_events.md +++ b/doc/api/resource_label_events.md @@ -1,12 +1,13 @@ --- stage: Manage -group: Compilance +group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Resource label events API +# Resource label events API **(FREE)** -Resource label events keep track about who, when, and which label was added to, or removed from, an issuable. +Resource label events keep track about who, when, and which label was added to (or removed from) +an issue, merge request, or epic. ## Issues diff --git a/doc/api/runners.md b/doc/api/runners.md index 1f0209c3cae..951e72edcb5 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -41,6 +41,7 @@ GET /runners?scope=active GET /runners?type=project_type GET /runners?status=active GET /runners?tag_list=tag1,tag2 +GET /runners?search=gitlab ``` | Attribute | Type | Required | Description | @@ -49,6 +50,7 @@ GET /runners?tag_list=tag1,tag2 | `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | | `status` | string | no | The status of runners to show, one of: `active`, `paused`, `online`, `offline` | | `tag_list` | string array | no | List of the runner's tags | +| `search` | string | no | The full token or partial description text to match | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/runners" @@ -62,8 +64,9 @@ Example response: "active": true, "description": "test-1-20150125", "id": 6, - "is_shared": false, "ip_address": "127.0.0.1", + "is_shared": false, + "runner_type": "project_type", "name": null, "online": true, "status": "online" @@ -74,6 +77,7 @@ Example response: "id": 8, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "group_type", "name": null, "online": false, "status": "offline" @@ -115,6 +119,7 @@ Example response: "id": 1, "ip_address": "127.0.0.1", "is_shared": true, + "runner_type": "instance_type", "name": null, "online": true, "status": "online" @@ -125,6 +130,7 @@ Example response: "id": 3, "ip_address": "127.0.0.1", "is_shared": true, + "runner_type": "instance_type", "name": null, "online": false, "status": "offline" @@ -135,6 +141,7 @@ Example response: "id": 6, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "project_type", "name": null, "online": true, "status": "paused" @@ -145,6 +152,7 @@ Example response: "id": 8, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "group_type", "name": null, "online": false, "status": "offline" @@ -156,7 +164,8 @@ Example response: Get details of a runner. -[Maintainer access or higher](../user/permissions.md) is required to get runner details at the project and group level. +The [Maintainer role or higher](../user/permissions.md) is required to get runner details at the +project and group level. Instance-level runner details via this endpoint are available to all signed in users. @@ -186,6 +195,7 @@ Example response: "id": 6, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "project_type", "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, "online": true, @@ -231,7 +241,8 @@ PUT /runners/:id | `maximum_timeout` | integer | no | Maximum timeout set when this runner handles the job | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/runners/6" --form "description=test-1-20150125-test" --form "tag_list=ruby,mysql,tag1,tag2" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/runners/6" \ + --form "description=test-1-20150125-test" --form "tag_list=ruby,mysql,tag1,tag2" ``` NOTE: @@ -248,6 +259,7 @@ Example response: "id": 6, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "group_type", "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, "online": true, @@ -288,7 +300,8 @@ PUT --form "active=false" /runners/:runner_id | `runner_id` | integer | yes | The ID of a runner | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --form "active=false" "https://gitlab.example.com/api/v4/runners/6" +curl --request PUT --header "PRIVATE-TOKEN: " \ + --form "active=false" "https://gitlab.example.com/api/v4/runners/6" ``` ## List runner's jobs @@ -417,6 +430,7 @@ Example response: "id": 8, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "project_type", "name": null, "online": false, "status": "offline" @@ -427,6 +441,7 @@ Example response: "id": 5, "ip_address": "127.0.0.1", "is_shared": true, + "runner_type": "instance_type", "name": null, "online": true, "status": "paused" @@ -448,7 +463,8 @@ POST /projects/:id/runners | `runner_id` | integer | yes | The ID of a runner | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/9/runners" --form "runner_id=9" +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/9/runners" \ + --form "runner_id=9" ``` Example response: @@ -460,6 +476,7 @@ Example response: "id": 9, "ip_address": "127.0.0.1", "is_shared": false, + "runner_type": "project_type", "name": null, "online": true, "status": "online" @@ -518,6 +535,7 @@ Example response: "ip_address": "127.0.0.1", "active": true, "is_shared": true, + "runner_type": "instance_type", "name": "gitlab-runner", "online": null, "status": "not_connected" @@ -528,6 +546,7 @@ Example response: "ip_address": "127.0.0.1", "active": true, "is_shared": true, + "runner_type": "instance_type", "name": "gitlab-runner", "online": false, "status": "offline" @@ -538,6 +557,7 @@ Example response: "ip_address": "127.0.0.1", "active": true, "is_shared": false, + "runner_type": "group_type", "name": "gitlab-runner", "online": null, "status": "not_connected" @@ -566,7 +586,9 @@ POST /runners | `maximum_timeout` | integer | no | Maximum timeout set when this runner handles the job | ```shell -curl --request POST "https://gitlab.example.com/api/v4/runners" --form "token=" --form "description=test-1-20150125-test" --form "tag_list=ruby,mysql,tag1,tag2" +curl --request POST "https://gitlab.example.com/api/v4/runners" \ + --form "token=" --form "description=test-1-20150125-test" \ + --form "tag_list=ruby,mysql,tag1,tag2" ``` Response: @@ -620,7 +642,8 @@ DELETE /runners | `token` | string | yes | The runner's [authentication token](#registration-and-authentication-tokens). | ```shell -curl --request DELETE "https://gitlab.example.com/api/v4/runners" --form "token=" +curl --request DELETE "https://gitlab.example.com/api/v4/runners" \ + --form "token=" ``` Response: @@ -642,7 +665,8 @@ POST /runners/verify | `token` | string | yes | Runner's [authentication token](#registration-and-authentication-tokens). | ```shell -curl --request POST "https://gitlab.example.com/api/v4/runners/verify" --form "token=" +curl --request POST "https://gitlab.example.com/api/v4/runners/verify" \ + --form "token=" ``` Response: diff --git a/doc/api/scim.md b/doc/api/scim.md index d00a0988d2b..42580ba65b6 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -39,7 +39,9 @@ Pagination follows the [SCIM spec](https://tools.ietf.org/html/rfc7644#section-3 Example request: ```shell -curl "https://gitlab.example.com/api/scim/v2/groups/test_group/Users?filter=id%20eq%20%220b1d561c-21ff-4092-beab-8154b17f82f2%22" --header "Authorization: Bearer " --header "Content-Type: application/scim+json" +curl "https://gitlab.example.com/api/scim/v2/groups/test_group/Users?filter=id%20eq%20%220b1d561c-21ff-4092-beab-8154b17f82f2%22" \ + --header "Authorization: Bearer " \ + --header "Content-Type: application/scim+json" ``` Example response: @@ -90,7 +92,8 @@ Parameters: Example request: ```shell -curl "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer " --header "Content-Type: application/scim+json" +curl "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" \ + --header "Authorization: Bearer " --header "Content-Type: application/scim+json" ``` Example response: @@ -134,7 +137,9 @@ Parameters: Example request: ```shell -curl --verbose --request POST "https://gitlab.example.com/api/scim/v2/groups/test_group/Users" --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' --header "Authorization: Bearer " --header "Content-Type: application/scim+json" +curl --verbose --request POST "https://gitlab.example.com/api/scim/v2/groups/test_group/Users" \ + --data '{"externalId":"test_uid","active":null,"userName":"username","emails":[{"primary":true,"type":"work","value":"name@example.com"}],"name":{"formatted":"Test User","familyName":"User","givenName":"Test"},"schemas":["urn:ietf:params:scim:schemas:core:2.0:User"],"meta":{"resourceType":"User"}}' \ + --header "Authorization: Bearer " --header "Content-Type: application/scim+json" ``` Example response: @@ -188,7 +193,9 @@ Parameters: Example request: ```shell -curl --verbose --request PATCH "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' --header "Authorization: Bearer " --header "Content-Type: application/scim+json" +curl --verbose --request PATCH "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" \ + --data '{ "Operations": [{"op":"Add","path":"name.formatted","value":"New Name"}] }' \ + --header "Authorization: Bearer " --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. @@ -211,7 +218,8 @@ Parameters: Example request: ```shell -curl --verbose --request DELETE "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" --header "Authorization: Bearer " --header "Content-Type: application/scim+json" +curl --verbose --request DELETE "https://gitlab.example.com/api/scim/v2/groups/test_group/Users/f0b1d561c-21ff-4092-beab-8154b17f82f2" \ + --header "Authorization: Bearer " --header "Content-Type: application/scim+json" ``` Returns an empty response with a `204` status code if successful. diff --git a/doc/api/search.md b/doc/api/search.md index c8f24c0924a..9714bc17f62 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -54,7 +54,8 @@ Example response: "path_with_namespace": "twitter/flight", "created_at": "2017-09-05T07:58:01.621Z", "default_branch": "master", - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo": "ssh://jarka@localhost:2222/twitter/flight.git", "http_url_to_repo": "http://localhost:3000/twitter/flight.git", "web_url": "http://localhost:3000/twitter/flight", @@ -177,6 +178,7 @@ Example response: "ruby", "tests" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 13, @@ -475,7 +477,8 @@ Example response: "path_with_namespace": "twitter/flight", "created_at": "2017-09-05T07:58:01.621Z", "default_branch": "master", - "tag_list":[], + "tag_list":[], //deprecated, use `topics` instead + "topics":[], "ssh_url_to_repo": "ssh://jarka@localhost:2222/twitter/flight.git", "http_url_to_repo": "http://localhost:3000/twitter/flight.git", "web_url": "http://localhost:3000/twitter/flight", @@ -598,6 +601,7 @@ Example response: "ruby", "tests" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 13, @@ -956,6 +960,7 @@ Example response: "ruby", "tests" ], + "draft": false, "work_in_progress": false, "milestone": { "id": 13, diff --git a/doc/api/services.md b/doc/api/services.md index e658c51f7e6..0a26a88de70 100644 --- a/doc/api/services.md +++ b/doc/api/services.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Services API **(FREE)** NOTE: -This API requires an access token with Maintainer or Owner permissions. +This API requires an access token with the [Maintainer or Owner role](../user/permissions.md). ## List all active services @@ -754,7 +754,7 @@ Set Jira service for a project. > Starting with GitLab 8.14, `api_url`, `issues_url`, `new_issue_url` and > `project_url` are replaced by `url`. If you are using an -> older version, [follow this documentation](https://gitlab.com/gitlab-org/gitlab/blob/8-13-stable-ee/doc/api/services.md#jira). +> older version, [follow this documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/8-13-stable-ee/doc/api/services.md#jira). ```plaintext PUT /projects/:id/services/jira diff --git a/doc/api/settings.md b/doc/api/settings.md index ada1d0e7fc4..8225713fc00 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -241,16 +241,19 @@ listed in the descriptions of the relevant settings. | `check_namespace_plan` | boolean | no | **(PREMIUM)** Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes. | +| `deactivate_dormant_users` | boolean | no | Enable [atomatic deactivation of dormant users](../user/admin_area/moderate_users.md#automatically-deactivate-dormant-users). | | `default_artifacts_expire_in` | string | no | Set the default expiration time for each job's artifacts. | | `default_branch_protection` | integer | no | Determine if developers can push to the default branch. Can take: `0` _(not protected, both developers and maintainers can push new commits, force push, or delete the branch)_, `1` _(partially protected, developers and maintainers can push new commits, but cannot force push, or delete, the branch)_ or `2` _(fully protected, developers cannot push new commits, but maintainers can; no-one can force push or delete the branch)_ as a parameter. Default is `2`. | -| `default_ci_config_path` | string | no | Default CI configuration path for new projects (`.gitlab-ci.yml` if not set). | +| `default_ci_config_path` | string | no | Default CI/CD configuration file and path for new projects (`.gitlab-ci.yml` if not set). | | `default_group_visibility` | string | no | What visibility level new groups receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_project_creation` | integer | no | Default project creation protection. Can take: `0` _(No one)_, `1` _(Maintainers)_ or `2` _(Developers + Maintainers)_| | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `deletion_adjourned_period` | integer | no | **(PREMIUM SELF)** The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. -| `diff_max_patch_bytes` | integer | no | Maximum diff patch size (Bytes). | +| `diff_max_patch_bytes` | integer | no | Maximum [diff patch size](../user/admin_area/diff_limits.md), in bytes. | +| `diff_max_files` | integer | no | Maximum [files in a diff](../user/admin_area/diff_limits.md). | +| `diff_max_lines` | integer | no | Maximum [lines in a diff](../user/admin_area/diff_limits.md). | | `disable_feed_token` | boolean | no | Disable display of RSS/Atom and calendar feed tokens ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231493) in GitLab 13.7) | | `disabled_oauth_sign_in_sources` | array of strings | no | Disabled OAuth sign-in sources. | | `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS rebinding attack protection. | diff --git a/doc/api/snippet_repository_storage_moves.md b/doc/api/snippet_repository_storage_moves.md index cc7f703f334..3454073cc8c 100644 --- a/doc/api/snippet_repository_storage_moves.md +++ b/doc/api/snippet_repository_storage_moves.md @@ -230,8 +230,10 @@ Supported attributes: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"destination_storage_name":"storage2"}' "https://gitlab.example.com/api/v4/snippets/1/repository_storage_moves" +curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"destination_storage_name":"storage2"}' \ + "https://gitlab.example.com/api/v4/snippets/1/repository_storage_moves" ``` Example response: @@ -279,8 +281,10 @@ Supported attributes: Example request: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ ---data '{"source_storage_name":"default"}' "https://gitlab.example.com/api/v4/snippet_repository_storage_moves" +curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"default"}' \ + "https://gitlab.example.com/api/v4/snippet_repository_storage_moves" ``` Example response: diff --git a/doc/api/status_checks.md b/doc/api/status_checks.md new file mode 100644 index 00000000000..f4e384a2efb --- /dev/null +++ b/doc/api/status_checks.md @@ -0,0 +1,201 @@ +--- +stage: Manage +group: Compliance +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" +type: reference, api +--- + +# External Status Checks API **(ULTIMATE)** + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3869) in GitLab 14.0. +> - It's [deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - It's disabled on GitLab.com. +> - It's not recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-status-checks). **(ULTIMATE SELF)** + +WARNING: +This feature might not be available to you. Check the **version history** note above for details. + +## List status checks for a merge request + +For a single merge request, list the external status checks that apply to it and their status. + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/status_checks +``` + +**Parameters:** + +| Attribute | Type | Required | Description | +| ------------------------ | ------- | -------- | -------------------------- | +| `id` | integer | yes | ID of a project | +| `merge_request_iid` | integer | yes | IID of a merge request | + +```json +[ + { + "id": 2, + "name": "Rule 1", + "external_url": "https://gitlab.com/test-endpoint", + "status": "approved" + }, + { + "id": 1, + "name": "Rule 2", + "external_url": "https://gitlab.com/test-endpoint-2", + "status": "pending" + } +] +``` + +## Set approval status of an external status check + +For a single merge request, use the API to inform GitLab that a merge request has been approved by an external service. + +```plaintext +POST /projects/:id/merge_requests/:merge_request_iid/status_check_responses +``` + +**Parameters:** + +| Attribute | Type | Required | Description | +| ------------------------ | ------- | -------- | -------------------------------------- | +| `id` | integer | yes | ID of a project | +| `merge_request_iid` | integer | yes | IID of a merge request | +| `sha` | string | yes | SHA at `HEAD` of the source branch | + +NOTE: +`sha` must be the SHA at the `HEAD` of the merge request's source branch. + +## Enable or disable status checks **(ULTIMATE SELF)** + +Status checks are under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can enable it. + +## Get project external status checks **(ULTIMATE)** + +You can request information about a project's external status checks using the following endpoint: + +```plaintext +GET /projects/:id/external_status_checks +``` + +**Parameters:** + +| Attribute | Type | Required | Description | +|---------------------|---------|----------|---------------------| +| `id` | integer | yes | The ID of a project | + +```json +[ + { + "id": 1, + "name": "Compliance Check", + "project_id": 6, + "external_url": "https://gitlab.com/example/test.json", + "protected_branches": [ + { + "id": 14, + "project_id": 6, + "name": "master", + "created_at": "2020-10-12T14:04:50.787Z", + "updated_at": "2020-10-12T14:04:50.787Z", + "code_owner_approval_required": false + } + ] + } +] +``` + +### Create external status check **(ULTIMATE)** + +You can create a new external status check for a project using the following endpoint: + +```plaintext +POST /projects/:id/external_status_checks +``` + +| Attribute | Type | Required | Description | +|------------------------|----------------|----------|----------------------------------------------------| +| `id` | integer | yes | The ID of a project | +| `name` | string | yes | Display name of status check | +| `external_url` | string | yes | URL of status check resource | +| `protected_branch_ids` | `array` | no | The ids of protected branches to scope the rule by | + +### Delete external status check **(ULTIMATE)** + +You can delete an external status check for a project using the following endpoint: + +```plaintext +DELETE /projects/:id/external_status_checks/:check_id +``` + +| Attribute | Type | Required | Description | +|------------------------|----------------|----------|----------------------------------------------------| +| `rule_id` | integer | yes | The ID of an status check | +| `id` | integer | yes | The ID of a project | + +### Update external status check **(ULTIMATE)** + +You can update an existing external status check for a project using the following endpoint: + +```plaintext +PUT /projects/:id/external_status_checks/:check_id +``` + +| Attribute | Type | Required | Description | +|------------------------|----------------|----------|----------------------------------------------------| +| `id` | integer | yes | The ID of a project | +| `rule_id` | integer | yes | The ID of an external status check | +| `name` | string | no | Display name of status check | +| `external_url` | string | no | URL of external status check resource | +| `protected_branch_ids` | `array` | no | The ids of protected branches to scope the rule by | + +### Enable or disable External Project-level MR status checks **(ULTIMATE SELF)** + +Enable or disable External Project-level MR status checks is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../user/feature_flags.md) +can enable it. + +To enable it: + +```ruby +# For the instance +Feature.enable(:ff_compliance_approval_gates) +# For a single project +Feature.enable(:ff_compliance_approval_gates, Project.find()) +``` + +To disable it: + +```ruby +# For the instance +Feature.disable(:ff_compliance_approval_gates) +# For a single project +Feature.disable(:ff_compliance_approval_gates, Project.find()) +``` + +To enable it: + +```ruby +# For the instance +Feature.enable(:ff_compliance_approval_gates) +# For a single project +Feature.enable(:ff_compliance_approval_gates, Project.find()) +``` + +To disable it: + +```ruby +# For the instance +Feature.disable(:ff_compliance_approval_gates) +# For a single project +Feature.disable(:ff_compliance_approval_gates, Project.find() +``` + +## Related links + +- [External status checks](../user/project/merge_requests/status_checks.md) diff --git a/doc/api/suggestions.md b/doc/api/suggestions.md index 0fcb6122505..ea9baa79b4a 100644 --- a/doc/api/suggestions.md +++ b/doc/api/suggestions.md @@ -7,6 +7,8 @@ type: reference, api # Suggest Changes API **(FREE)** +This page describes the API for [suggesting changes](../user/project/merge_requests/reviews/suggestions.md). + Every API call to suggestions must be authenticated. ## Applying suggestions diff --git a/doc/api/system_hooks.md b/doc/api/system_hooks.md index 101769e6323..1f0bce1c78f 100644 --- a/doc/api/system_hooks.md +++ b/doc/api/system_hooks.md @@ -8,8 +8,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w All methods require administrator authorization. -The URL endpoint of the system hooks can also be configured using the UI in -the **Admin Area > System Hooks** (`/admin/hooks`). +You can configure the URL endpoint of the system hooks from the GitLab user interface: + +1. On the top bar, select **Menu >** **{admin}** **Admin**. +1. Select **System Hooks** (`/admin/hooks`). Read more about [system hooks](../system_hooks/system_hooks.md). diff --git a/doc/api/tags.md b/doc/api/tags.md index 3ac4e8bb6ab..53a981256aa 100644 --- a/doc/api/tags.md +++ b/doc/api/tags.md @@ -123,7 +123,6 @@ Parameters: | `tag_name` | string | yes | The name of a tag | | `ref` | string | yes | Create tag using commit SHA, another tag name, or branch name | | `message` | string | no | Creates annotated tag | -| `release_description` | string | no | This parameter is [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) for use in GitLab 11.7, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/290311) in GitLab 14.0. Use the [Releases API](../api/releases/index.md) instead. | ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/repository/tags?tag_name=test&ref=master" @@ -149,10 +148,7 @@ Example response: "committer_email": "jack@example.com", "committed_date": "2012-05-28T04:42:42-07:00" }, - "release": { - "tag_name": "1.0.0", - "description": "Amazing release. Wow" - }, + "release": null, "name": "v1.0.0", "target": "2695effb5807a22ff3d138d593fd856244e155e7", "message": null, @@ -182,82 +178,3 @@ Parameters: | ---------- | -------------- | -------- | --------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `tag_name` | string | yes | The name of a tag | - -## Create a new release - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) -for use in GitLab 11.7, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/290311) -in GitLab 14.0. Use the [Releases API](../api/releases/index.md) instead. - -Add release notes to the existing Git tag. If there -already exists a release for the given tag, status code `409` is returned. - -```plaintext -POST /projects/:id/repository/tags/:tag_name/release -``` - -Parameters: - -| Attribute | Type | Required | Description | -| ---------- | -------------- | -------- | --------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `tag_name` | string | yes | The name of a tag | - -Request body: - -- `description` (required) - Release notes with Markdown support - -```json -{ - "description": "Amazing release. Wow" -} -``` - -Response: - -```json -{ - "tag_name": "1.0.0", - "description": "Amazing release. Wow" -} -``` - -## Update a release - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41766) -for use in GitLab 11.7, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/290311) -in GitLab 14.0. Use the [Releases API](../api/releases/index.md) instead. - -Updates the release notes of a given release. - -```plaintext -PUT /projects/:id/repository/tags/:tag_name/release -``` - -Parameters: - -| Attribute | Type | Required | Description | -| ---------- | -------------- | -------- | --------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | -| `tag_name` | string | yes | The name of a tag | - -Request body: - -- `description` (required) - Release notes with Markdown support - -```json -{ - "description": "Amazing release. Wow" -} -``` - -Response: - -```json -{ - "tag_name": "1.0.0", - "description": "Amazing release. Wow" -} -``` diff --git a/doc/api/templates/dockerfiles.md b/doc/api/templates/dockerfiles.md index 6eedf8d2bc0..2d7e926561f 100644 --- a/doc/api/templates/dockerfiles.md +++ b/doc/api/templates/dockerfiles.md @@ -74,10 +74,6 @@ Example response: "key": "OpenJDK", "name": "OpenJDK" }, - { - "key": "OpenJDK-alpine", - "name": "OpenJDK-alpine" - }, { "key": "PHP", "name": "PHP" diff --git a/doc/api/templates/gitlab_ci_ymls.md b/doc/api/templates/gitlab_ci_ymls.md index 9475febdaec..388556d354f 100644 --- a/doc/api/templates/gitlab_ci_ymls.md +++ b/doc/api/templates/gitlab_ci_ymls.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference --- diff --git a/doc/api/todos.md b/doc/api/todos.md index 864f87f988e..69c0f760a29 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -88,6 +88,7 @@ Example Response: "source_project_id": 2, "target_project_id": 2, "labels": [], + "draft": false, "work_in_progress": false, "milestone": { "id": 32, @@ -161,6 +162,7 @@ Example Response: "source_project_id": 2, "target_project_id": 2, "labels": [], + "draft": false, "work_in_progress": false, "milestone": { "id": 32, @@ -259,6 +261,7 @@ Example Response: "source_project_id": 2, "target_project_id": 2, "labels": [], + "draft": false, "work_in_progress": false, "milestone": { "id": 32, diff --git a/doc/api/users.md b/doc/api/users.md index ac8fbe8492f..0e7b197b106 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -207,7 +207,7 @@ Users on GitLab [Premium or higher](https://about.gitlab.com/pricing/) also see ``` Users on GitLab [Premium or higher](https://about.gitlab.com/pricing/) also see -the `group_saml` provider option: +the `group_saml` provider option and `provisioned_by_group_id` parameter: ```json [ @@ -220,6 +220,7 @@ the `group_saml` provider option: {"provider": "google_oauth2", "extern_uid": "8776128412476123468721346"}, {"provider": "group_saml", "extern_uid": "123789", "saml_provider_id": 10} ], + "provisioned_by_group_id": 123789 ... } ] @@ -374,7 +375,7 @@ the `shared_runners_minutes_limit`, `is_auditor`, and `extra_shared_runners_minu ``` Users on GitLab.com [Premium or higher](https://about.gitlab.com/pricing/) also -see the `group_saml` option: +see the `group_saml` option and `provisioned_by_group_id` parameter: ```json { @@ -388,6 +389,7 @@ see the `group_saml` option: {"provider": "google_oauth2", "extern_uid": "8776128412476123468721346"}, {"provider": "group_saml", "extern_uid": "123789", "saml_provider_id": 10} ], + "provisioned_by_group_id": 123789 ... } ``` @@ -630,7 +632,14 @@ GET /user } ``` -Users on GitLab [Premium or higher](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, `is_auditor`, and `using_license_seat` parameters. +Users on GitLab [Premium or higher](https://about.gitlab.com/pricing/) also see these +parameters: + +- `shared_runners_minutes_limit` +- `extra_shared_runners_minutes_limit` +- `is_auditor` +- `provisioned_by_group_id` +- `using_license_seat` ## User status @@ -684,6 +693,29 @@ Example response: } ``` +## Get user preferences + +Get a list of currently authenticated user's preferences. + +```plaintext +GET /user/preferences +``` + +Example response: + +```json +{ + "id": 1, + "user_id": 1 + "view_diffs_file_by_file": true, + "show_whitespace_in_diffs": false +} +``` + +Parameters: + +- **none** + ## User preference modification Update the current user's preferences. @@ -696,7 +728,8 @@ PUT /user/preferences { "id": 1, "user_id": 1 - "view_diffs_file_by_file": true + "view_diffs_file_by_file": true, + "show_whitespace_in_diffs": false } ``` @@ -705,6 +738,7 @@ Parameters: | Attribute | Required | Description | | :--------------------------- | :------- | :---------------------------------------------------------- | | `view_diffs_file_by_file` | Yes | Flag indicating the user sees only one file diff per page. | +| `show_whitespace_in_diffs` | Yes | Flag indicating the user sees whitespace changes in diffs. | ## Set user status @@ -723,7 +757,8 @@ PUT /user/status When both parameters `emoji` and `message` are empty, the status is cleared. When the `clear_status_after` parameter is missing from the request, the previously set value for `"clear_status_after` is cleared. ```shell -curl --request PUT --header "PRIVATE-TOKEN: " --data "clear_status_after=1_day" --data "emoji=coffee" --data "message=I crave coffee" "https://gitlab.example.com/api/v4/user/status" +curl --request PUT --header "PRIVATE-TOKEN: " --data "clear_status_after=1_day" --data "emoji=coffee" \ + --data "message=I crave coffee" "https://gitlab.example.com/api/v4/user/status" ``` Example responses @@ -1058,7 +1093,8 @@ Parameters: | key | string | yes | The new GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/user/gpg_keys" +curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/user/gpg_keys" ``` Example response: @@ -1169,7 +1205,8 @@ Parameters: | `key_id` | integer | yes | The ID of the GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/users/2/gpg_keys" +curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/users/2/gpg_keys" ``` Example response: @@ -1579,7 +1616,8 @@ POST /users/:user_id/impersonation_tokens | `scopes` | array | yes | The array of scopes of the impersonation token (`api`, `read_user`) | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "name=mytoken" --data "expires_at=2017-04-04" --data "scopes[]=api" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens" +curl --request POST --header "PRIVATE-TOKEN: " --data "name=mytoken" --data "expires_at=2017-04-04" \ + --data "scopes[]=api" "https://gitlab.example.com/api/v4/users/42/impersonation_tokens" ``` Example response: @@ -1643,7 +1681,8 @@ POST /users/:user_id/personal_access_tokens | `scopes` | array | yes | The array of scopes of the personal access token (`api`, `read_user`, `read_api`, `read_repository`, `write_repository`) | ```shell -curl --request POST --header "PRIVATE-TOKEN: " --data "name=mytoken" --data "expires_at=2017-04-04" --data "scopes[]=api" "https://gitlab.example.com/api/v4/users/42/personal_access_tokens" +curl --request POST --header "PRIVATE-TOKEN: " --data "name=mytoken" --data "expires_at=2017-04-04" \ + --data "scopes[]=api" "https://gitlab.example.com/api/v4/users/42/personal_access_tokens" ``` Example response: diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index f70662c7c61..6d01d7e7d96 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -199,13 +199,13 @@ Example response: ```csv Group Name,Project Name,Scanner Type,Scanner Name,Status,Vulnerability,Details,Additional Info,Severity,CVE,CWE,Other Identifiers -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520 -Gitlab.org,Defend,container_scanning,Clair,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869,CWE-1 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520 +Gitlab.org,Defend,container_scanning,Trivy,detected,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869,CWE-1 Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,CVE-2021-1234,CWE-2,"""yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a""" Gitlab.org,Defend,dependency_scanning,Gemnasium,detected,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,,,"""yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98""" Gitlab.org,Defend,sast,Find Security Bugs,detected,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,,,"""818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47""" diff --git a/doc/api/wikis.md b/doc/api/wikis.md index 569708cdfcc..d6cc6f938b8 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -64,7 +64,7 @@ GET /projects/:id/wikis/:slug | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `slug` | string | yes | The slug (a unique string) of the wiki page | +| `slug` | string | yes | URLencoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis/home" @@ -97,7 +97,8 @@ POST /projects/:id/wikis | `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | ```shell -curl --data "format=rdoc&title=Hello&content=Hello world" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis" +curl --data "format=rdoc&title=Hello&content=Hello world" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis" ``` Example response: @@ -125,10 +126,11 @@ PUT /projects/:id/wikis/:slug | `content` | string | yes if `title` is not provided | The content of the wiki page | | `title` | string | yes if `content` is not provided | The title of the wiki page | | `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | -| `slug` | string | yes | The slug (a unique string) of the wiki page | +| `slug` | string | yes | URL-encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | ```shell -curl --request PUT --data "format=rdoc&content=documentation&title=Docs" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis/foo" +curl --request PUT --data "format=rdoc&content=documentation&title=Docs" \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis/foo" ``` Example response: @@ -153,7 +155,7 @@ DELETE /projects/:id/wikis/:slug | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `slug` | string | yes | The slug (a unique string) of the wiki page | +| `slug` | string | yes | URL-encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/wikis/foo" @@ -182,7 +184,8 @@ The `file=` parameter must point to a file on your file system and be preceded by `@`. For example: ```shell -curl --request POST --header "PRIVATE-TOKEN: " --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/1/wikis/attachments" +curl --request POST --header "PRIVATE-TOKEN: " \ + --form "file=@dk.png" "https://gitlab.example.com/api/v4/projects/1/wikis/attachments" ``` Example response: diff --git a/doc/architecture/blueprints/ci_scale/ci_builds_cumulative_forecast.png b/doc/architecture/blueprints/ci_scale/ci_builds_cumulative_forecast.png index fa34c7d1c36..d1e7db30b11 100644 Binary files a/doc/architecture/blueprints/ci_scale/ci_builds_cumulative_forecast.png and b/doc/architecture/blueprints/ci_scale/ci_builds_cumulative_forecast.png differ diff --git a/doc/architecture/blueprints/ci_scale/ci_builds_daily_forecast.png b/doc/architecture/blueprints/ci_scale/ci_builds_daily_forecast.png index b73a592fa6b..15f250e6b0e 100644 Binary files a/doc/architecture/blueprints/ci_scale/ci_builds_daily_forecast.png and b/doc/architecture/blueprints/ci_scale/ci_builds_daily_forecast.png differ diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index 99997e7b19b..6afa13bf207 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -28,7 +28,7 @@ store CI/CD data. We expect to see 20M builds created daily on GitLab.com in the first half of 2024. -![ci_builds cumulative with forecast](ci_builds_cumulative_forecast.png) +![CI builds cumulative with forecast](ci_builds_cumulative_forecast.png) ## Goals @@ -46,9 +46,9 @@ Historically, Rails used to use [integer](https://www.postgresql.org/docs/9.1/da type when creating primary keys for a table. We did use the default when we [created the `ci_builds` table in 2012](https://gitlab.com/gitlab-org/gitlab/-/blob/046b28312704f3131e72dcd2dbdacc5264d4aa62/db/ci/migrate/20121004165038_create_builds.rb). [The behavior of Rails has changed](https://github.com/rails/rails/pull/26266) -since the release of Rails 5. The framework is now using bigint type that is 8 +since the release of Rails 5. The framework is now using `bigint` type that is 8 bytes long, however we have not migrated primary keys for `ci_builds` table to -bigint yet. +`bigint` yet. We will run out of the capacity of the integer type to store primary keys in `ci_builds` table before December 2021. When it happens without a viable @@ -89,7 +89,7 @@ Prophet](https://facebook.github.io/prophet/) shows that in the first half of to around 2M we see created today, this is 10x growth our product might need to sustain in upcoming years. -![ci_builds daily forecast](ci_builds_daily_forecast.png) +![CI builds daily forecast](ci_builds_daily_forecast.png) ### Queuing mechanisms are using the large table @@ -101,7 +101,7 @@ want to process them. This mechanism is very inefficient, and it has been causing problems on the production environment frequently. This usually results in a significant drop -of the CI/CD apdex score, and sometimes even causes a significant performance +of the CI/CD Apdex score, and sometimes even causes a significant performance degradation in the production environment. There are multiple other strategies that can improve performance and diff --git a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md new file mode 100644 index 00000000000..92717dc1fe9 --- /dev/null +++ b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md @@ -0,0 +1,614 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +comments: false +description: 'Making a GitLab codebase composable - allowing to run parts of the application' +--- + +# Composable GitLab codebase - using Rails Engines + +The one of the major risks of a single codebase is an infinite growth of the whole +application. The more code being added results in not only ever increasing resource requirements +for running the application, but increased application coupling and explosion of the complexity. + +## Executive summary + +This blueprint discusses an impact of introducing **Application Layers** as a way to reduce and improve the application +codebase. This discusses the positive and negative outcomes of the proposed solution and tries to estimate the impact +on GitLab.com and smaller installations. + +**Application Layers** tries to split GitLab Rails codebase horizontally following the pattern of how we actually +run GitLab instead of vertical split. This follows the idea that a single feature needs to run in many different ways +(CI for example has Web interface, uses API, and performs background processing), and we are not able to easily +run only a given feature separate to the rest application (like CI) due to coupling. + +The proposal itself does allow us to disconnect some aspects of the features. These aspects could be treated +as components that are run separately from the rest of the stack, but still sharing a large portion of core. +This model could be implemented to provide an API interface for external tooling (Runners API, Packages API, Feature Flags Unleash API) +and allow us to have much better resiliency and much easier way to scale application in the future. + +The actual split was tested with the usage of [Rails Engines](https://guides.rubyonrails.org/engines.html) +implementing separate gems in a single repository. The [Rails Engines](https://guides.rubyonrails.org/engines.html) +allowed us to well describe the individual components with its dependencies and run an application +consisting of many Rails Engines. + +The blueprint aims to retain all key aspects of GitLab success: single and monolithic codebase (with a [single data-store](https://about.gitlab.com/handbook/product/single-application/#single-data-store)), +but allows us to better model application and make our codebase more composable. + +## Challenges of the Monolith (a current state) + +Today the usage of monolith proves to be challenging in many cases. A single big monolith +codebase without clear boundaries results in a number of problems and inefficiencies, some of them being: + +- Deep coupling makes application harder to develop in longer term, as it leads to a spaghetti implementation + instead of considering building more interface-based architecture +- Deep coupling between parts of the codebase making it harder to test. To test only a small portion of application + we usually need to run a whole test suite to confidently know which parts are affected. This to + some extent can be improved by building a heuristic to aid this process, but it is prone to errors and hard + to keep accurate at all times +- All components need to be loaded at all times in order to run only parts of the application +- Increased resource usage, as we load parts of the application that are rarely used in a given context +- The high memory usage results in slowing the whole application as it increases GC cycles duration + creating significantly longer latency for processing requests or worse cache usage of CPUs +- Increased application boot-up times as we need to load and parse significantly more files +- Longer boot-up times slows down the development, as running application or tests takes significantly longer + reducing velocity and amount of iterations + +## Composable codebase dimensions + +In general, we can think about two ways how codebase can be modeled: + +- **vertically** in Bounded Contexts, each representing a domain of the application, ex.: All features related to CI are in a given context +- **horizontally** in Application Layers: Sidekiq, GraphQL, REST API, Web Controllers, all Domain Models and Services that interface with DB directly + +This blueprint explicitly talks about **horizontal** split and **Application Layers**. + +## Current state of Bounded Contexts (**vertical** split) + +The Bounded Contexts is a topic that was discussed extensively number of times for a couple of years. +Reflected in number of issues: + +- [Create new models / classes within a module / namespace](https://gitlab.com/gitlab-org/gitlab/-/issues/212156) +- [Make teams to be maintainers of their code](https://gitlab.com/gitlab-org/gitlab/-/issues/25872) +- [Use nested structure to organize CI classes](https://gitlab.com/gitlab-org/gitlab/-/issues/209745) +- [WIP: Make it simple to build and use "Decoupled Services"](https://gitlab.com/gitlab-org/gitlab/-/issues/31121) + +We are partially executing a **Bounded Contexts** idea: + +- Make each team to own their own namespace, namespace which is defined as a `module` in a codebase +- Make each team to own their own tests, as namespaces would define a clear boundaries +- Since we use namespaces, individual contributor or reviewer can know who to reach from domain experts about help with + the given context + +The module namespaces are actively being used today to model codebase around team boundaries. Currently, the most +prominent namespaces being used today are `Ci::` and `Packages::`. They provide a good way to contain the code owned +by a group in a well-defined structure. + +However, the **Bounded Contexts** while it helps development, it does not help with the above stated goals. This is purely +a logical split of the code. This does not prevent deep-coupling. It is still possible to create a circular dependency (and it often happens) +between a background processing of CI pipeline and Runner API interface. +API can call Sidekiq Worker, Sidekiq can use API to create an endpoint path. + +The **Bounded Contexts** do not make our codebase smarter to know what depends on what, as the whole codebase +is treated as single package that needs to be loaded and executed. + +Possible additional considerations to the disadvantages of Bounded Context: + +- It can lead to tribal knowledge and duplicate code +- The deep coupling can make it difficult to iterate and make minimal changes +- Changes may have cascading effects that are difficult to isolate due to the vertical split + +## The Application Layers (**horizontal* split) + +While we continue leveraging **Bounded Contexts** in form of namespace separation that aids development and review process +the **Application Layers** can provide a way to create a clean separation between different functional parts. + +Our main codebase (`GitLab Rails` after a GitLab running on Ruby on Rails) consists many of implicit **Application Layers**. +There are no clear boundaries between each layer which results in a deep coupling. + +The concept of **Application Layers** looks at the application from the perspective of how we run the application +instead of perspective of individual features (like CI or Packages). GitLab application today can be decomposed into the following +application layers. This list is not exhaustive, but shows a general list of the different parts of a single monolithic codebase: + +- Web Controllers: process Web requests coming from users visiting web interface +- Web API: API calls coming from the automated tooling, in some cases also users visiting web interface +- Web Runners API: API calls from the Runners, that allows Runner to fetch new jobs, or update trace log +- Web GraphQL: provide a flexible API interface, allowing the Web frontend to fetch only the data needed thereby reducing the amount of compute and data transfer +- Web ActionCable: provide bi-directional connection to enable real-time features for Users visiting web interface +- Web Feature Flags Unleash Backend: provide an Unleash-compatible Server that uses GitLab API +- Web Packages API: provide a REST API compatible with the packaging tools: Debian, Maven, Container Registry Proxy, etc. +- Git nodes: all code required to authorize `git pull/push` over `SSH` or `HTTPS` +- Sidekiq: run background jobs +- Services/Models/DB: all code required to maintain our database structure, data validation, business logic and policies models that needs to be shared with other components + +The best way to likely describe how the actual GitLab Rails split would look like. It is a satellite model. +Where we have a single core, that is shared across all satellite components. The design of that implies +that satellite components have a limited way to communicate with each other. In a single monolithic application +in most of cases application would communicate with a code. In a satellite model the communication needs +to be performed externally to the component. This can be via Database, Redis or using a well defined exposed API. + +```mermaid +flowchart TD + subgraph Data Store + D[Database] + R[Redis] + end + subgraph Rails Engines + subgraph Data Access Layer + C[Core] + end + subgraph Web Processing + W[Web] + end + subgraph Background Processing + S[Sidekiq] + end + end + C --> D & R + W & S -- using application models --> C + R -- push background job --> S + W -- via async schedule --> S + S -- via Web API --> W +``` + +### Application Layers for on-premise installations + +The on-premise installations are significantly smaller and they usually run GitLab Rails in two main flavors: + +```mermaid +graph LR + gitlab_node[GitLab Node with Load Balancer] + + gitlab_node_web[Web running Puma] + gitlab_node_sidekiq[Background jobs running Sidekiq] + gitlab_node_git[Git running Puma and SSH] + + subgraph GitLab Rails + gitlab_rails_web_controllers[Controllers] + gitlab_rails_api[API] + gitlab_rails_api_runners[API Runner] + gitlab_rails_graphql[GraphQL] + gitlab_rails_actioncable[ActionCable] + gitlab_rails_services[Services] + gitlab_rails_models[Models] + gitlab_rails_sidekiq[Sidekiq Workers] + end + + postgresql_db[(PostgreSQL Database)] + redis_db[(Redis Database)] + + gitlab_node --> gitlab_node_web + gitlab_node --> gitlab_node_sidekiq + gitlab_node --> gitlab_node_git + + gitlab_node_web --> gitlab_rails_web_controllers + gitlab_node_web --> gitlab_rails_api + gitlab_node_web --> gitlab_rails_api_runners + gitlab_node_web --> gitlab_rails_graphql + gitlab_node_web --> gitlab_rails_actioncable + gitlab_node_git --> gitlab_rails_api + gitlab_node_sidekiq --> gitlab_rails_sidekiq + + gitlab_rails_web_controllers --> gitlab_rails_services + gitlab_rails_api --> gitlab_rails_services + gitlab_rails_api_runners --> gitlab_rails_services + gitlab_rails_graphql --> gitlab_rails_services + gitlab_rails_actioncable --> gitlab_rails_services + gitlab_rails_sidekiq --> gitlab_rails_services + + gitlab_rails_services --> gitlab_rails_models + + gitlab_rails_models --> postgresql_db + gitlab_rails_models --> redis_db +``` + +### Application Layers on GitLab.com + +Due to its scale, GitLab.com requires much more attention to run. This is needed in order to better manage resources +and provide SLAs for different functional parts. The chart below provides a simplistic view of GitLab.com application layers. +It does not include all components, like Object Storage nor Gitaly nodes, but shows the GitLab Rails dependencies between +different components and how they are configured on GitLab.com today: + +```mermaid +graph LR + gitlab_com_lb[GitLab.com Load Balancer] + + gitlab_com_web[Web Nodes running Puma] + gitlab_com_api[API Nodes running Puma] + gitlab_com_websockets[WebSockets Nodes running Puma] + gitlab_com_sidekiq[Background Jobs running Sidekiq] + gitlab_com_git[Git Nodes running Puma and SSH] + + subgraph GitLab Rails + gitlab_rails_web_controllers[Controllers] + gitlab_rails_api[API] + gitlab_rails_api_runners[API Runner] + gitlab_rails_graphql[GraphQL] + gitlab_rails_actioncable[ActionCable] + gitlab_rails_services[Services] + gitlab_rails_models[Models] + gitlab_rails_sidekiq[Sidekiq Workers] + end + + postgresql_db[(PostgreSQL Database)] + redis_db[(Redis Database)] + + gitlab_com_lb --> gitlab_com_web + gitlab_com_lb --> gitlab_com_api + gitlab_com_lb --> gitlab_com_websockets + gitlab_com_lb --> gitlab_com_git + + gitlab_com_web --> gitlab_rails_web_controllers + gitlab_com_api --> gitlab_rails_api + gitlab_com_api --> gitlab_rails_api_runners + gitlab_com_api --> gitlab_rails_graphql + gitlab_com_websockets --> gitlab_rails_actioncable + gitlab_com_git --> gitlab_rails_api + gitlab_com_sidekiq --> gitlab_rails_sidekiq + + gitlab_rails_web_controllers --> gitlab_rails_services + gitlab_rails_api --> gitlab_rails_services + gitlab_rails_api_runners --> gitlab_rails_services + gitlab_rails_graphql --> gitlab_rails_services + gitlab_rails_actioncable --> gitlab_rails_services + gitlab_rails_sidekiq --> gitlab_rails_services + + gitlab_rails_services --> gitlab_rails_models + + gitlab_rails_models --> postgresql_db + gitlab_rails_models --> redis_db +``` + +### Layer dependencies + +The differences in how GitLab is run for on-premise versus how we run GitLab.com does show a main division line in GitLab Rails: + +- Web: containing all API, all Controllers, all GraphQL and ActionCable functionality +- Sidekiq: containing all background processing jobs +- Core: containing all database, models and services that needs to be shared between Web and Sidekiq + +Each of these top-level application layers do depend only on a fraction of the codebase with all relevant dependencies: + +- In all cases we need the underlying database structure and application models +- In some cases we need dependent services +- We only need a part of the application common library +- We need gems to support the requested functionality +- Individual layers should not use another sibling layer (tight coupling), rather connect via API, Redis or DB to share data (loose coupling) + +## Proposal + +The Memory team group conducted a Proof-of-Concept phase to understand the impact of introducing **Application Layers**. +We did this to understand the complexity, impact, and needed iterations to execute this proposal. + +The proposals here should be treated as evaluation of the impact of this blueprint, +but not a final solution to be implemented. The PoC as defined is not something that should be merged, +but serves as a basis for future work. + +### PoC using Rails Engines + +We decided to use Rails Engines by modeling a Web Application Layer. The Web Engine contained Controllers, API, GraphQL. +This allowed us to run Web Nodes with all dependencies, but measure the impact on Sidekiq not having these components loaded. + +All work can be found in these merge requests: + +- [Provide mechanism to load GraphQL with all dependencies only when needed](https://gitlab.com/gitlab-org/gitlab/-/issues/288044) +- [Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) +- [Draft: PoC - Move Controllers and Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720) +- [Draft: PoC - Move only Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982) +- [Measure performance impact for proposed web_engine](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) + +What was done? + +- We used [Rails Engines](https://guides.rubyonrails.org/engines.html) +- The 99% of changes as visible in the above MRs is moving files as-is +- We moved all GraphQL code and specs into `engines/web_engine/` as-is +- We moved all API and Controllers code and specs into `engines/web_engine` +- We adapted CI to test `engines/web_engine/` as a self-sufficient component of stack +- We configured GitLab to load `gem web_engine` running Web nodes (Puma web server) +- We disabled loading `web_engine` when running Background processing nodes (Sidekiq) + +#### Implementation details for proposed solution + +1. Introduce new Rails Engine for each application layer. + + We created `engines` folder, which could contain different engines for each application layer we introduce in the future. + + In the above PoCs we introduced the new Web Application Layer, located in `engines/web_engine` folder. + +1. Move all code and specs into `engines/web_engine/` + + - We moved all GraphQL code and specs into `engines/web_engine/` without changing files itself + - We moved all Grape API and Controllers code into `engines/web_engine/` without changing files itself + +1. Move gems to the `engines/web_engine/` + + - We moved all GraphQL gems to the actual web_engine Gemfile + - We moved Grape API gem to the actual web_engine Gemfile + + ```ruby + Gem::Specification.new do |spec| + spec.add_dependency 'apollo_upload_server' + spec.add_dependency 'graphql' + spec.add_dependency 'graphiql-rails' + + spec.add_dependency 'graphql-docs' + spec.add_dependency 'grape' + end + ``` + +1. Move routes to the `engines/web_engine/config/routes.rb` file + + - We moved GraphQL routes to the web_engine routes. + - We moved API routes to the web_engine routes. + - We moved most of the controller routes to the web_engine routes. + + ```ruby + Rails.application.routes.draw do + post '/api/graphql', to: 'graphql#execute' + mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: + Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/graphql') + + draw :api + + #... + end + ``` + +1. Move initializers to the `engines/web_engine/config/initializers` folder + + - We moved `graphql.rb` initializer to the `web_engine` initializers folder + - We moved `grape_patch.rb` and `graphe_validators` to the `web_engine` initializers folder + +1. Connect GitLab application with the WebEngine + + In GitLab Gemfile.rb, add web_engine to the engines group + + ```ruby + # Gemfile + group :engines, :test do + gem 'web_engine', path: 'engines/web_engine' + end + ``` + + Since the gem is inside :engines group, it will not be automatically required by default. + +1. Configure GitLab when to load the engine. + + In GitLab `config/engines.rb`, we can configure when do we want to load our engines by relying on our `Gitlab::Runtime` + + ```ruby + # config/engines.rb + # Load only in case we are running web_server or rails console + if Gitlab::Runtime.web_server? || Gitlab::Runtime.console? + require 'web_engine' + end + ``` + +1. Configure Engine + + Our Engine inherits from the `Rails::Engine` class. This way this gem notifies Rails that + there's an engine at the specified path so it will correctly mount the engine inside + the application, performing tasks such as adding the app directory of the engine to + the load path for models, mailers, controllers, and views. + A file at `lib/web_engine/engine.rb`, is identical in function to a standard Rails + application's `config/application.rb` file. This way engines can access a configuration + object which contains configuration shared by all railties and the application. + Additionally, each engine can access `autoload_paths`, `eager_load_paths`, and `autoload_once_paths` + settings which are scoped to that engine. + + ```ruby + module WebEngine + class Engine < ::Rails::Engine + config.eager_load_paths.push(*%W[#{config.root}/lib + #{config.root}/app/graphql/resolvers/concerns + #{config.root}/app/graphql/mutations/concerns + #{config.root}/app/graphql/types/concerns]) + + if Gitlab.ee? + ee_paths = config.eager_load_paths.each_with_object([]) do |path, memo| + ee_path = config.root + .join('ee', Pathname.new(path).relative_path_from(config.root)) + memo << ee_path.to_s + end + # Eager load should load CE first + config.eager_load_paths.push(*ee_paths) + end + end + end + ``` + +1. Testing + + We adapted CI to test `engines/web_engine/` as a self-sufficient component of stack. + + - We moved `spec` as-is files to the `engines/web_engine/spec` folder + - We moved `ee/spec` as-is files to the `engines/web_engine/ee/spec` folder + - We control specs from main application using environment variable `TEST_WEB_ENGINE` + - We added new CI job that will run `engines/web_engine/spec` tests separately using `TEST_WEB_ENGINE` environment variable. + - We added new CI job that will run `engines/web_engine/ee/spec` tests separately using `TEST_WEB_ENGINE` environment variable. + - We are running all whitebox frontend tests with `TEST_WEB_ENGINE=true` + +#### Results + +The effect on introducing these changes: + +- Savings for RSS +- 61.06 MB (7.76%) - Sidekiq without GraphQL +- 100.11 MB (12.73%) - Sidekiq without GraphQL and API +- 208.83 MB (26.56%) - Sidekiq without GraphQL, API, Controllers +- The size of Web nodes (running Puma) stayed the same as before + +Savings on Sidekiq `start-up` event, for a single Sidekiq cluster without GraphQL, API, Controllers + +- We saved 264.13 MB RSS (28.69%) +- We saved 264.09 MB USS (29.36%) +- Boot-up time was reduced from 45.31 to 21.80 seconds. It was 23.51 seconds faster (51.89%) +- We have 805,772 less live objects, 4,587,535 less allocated objects, 2,866 less allocated pages and 3.65 MB less allocated space for objects outside of the heap +- We loaded 2,326 less code files (15.64%) +- We reduced the duration of a single full GC cycle from 0.80s to 0.70 (12.64%) + +Puma single, showed very little difference as expected. + +More details can be found in the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/300548#note_516323444). + +#### Impact on GitLab.com + +Estimating the results for the scale of running GitLab.com, today we use: + +- Currently individual GC cycle takes around [130ms for Web](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) + and [200ms for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) on GitLab.com +- On average we do around [2 GC cycles per-second](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.end_input=2021-02-17%2017%3A56&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_count%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D))&g0.tab=0) + or [0.12 cycles per second for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.end_input=2021-02-17%2017%3A56&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_count%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D))&g0.tab=0) +- This translates to using [around 9.5 vCPUs per-second for Web](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D))&g0.tab=0) + and [around 8 vCPUs per-second for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D))&g0.tab=0) of spend on GC alone +- Sidekiq [uses 2.1GB on average](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=max(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=1) + or [550GB in total](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=0) of memory on GitLab.com + +We estimate the possible maximum savings for introducing `web_engine`: + +- Reduce a GC cycle time by 20%, from to 200ms to 160ms +- The amount of GC cycles per-second would stay the same, but due to GC cycle time reduction we would use around 6 vCPUs instead of 8 vCPUs +- In the best case we would be looking at Sidekiq alone we would be estimating to save up-to 137GB of memory on GitLab.com + +This model could be extended to introduce `sidekiq_engine` giving a similar benefits +(even more important due to visible impact on users) for Web nodes. + +#### Outcome + +We achieved a number of benefits introducing these changes. + +Pros: + +- Significantly lower memory usage +- Significantly shorter application load time for Sidekiq +- Significantly improved responsiveness of Sidekiq service due to much shorter GC cycles +- Significantly easier testing of a portion of application, ex. changing `web_engines/` does require + re-running test only for this application layer +- We retained a monolithic architecture of the codebase, but sharing database and application models +- A significant saving from the infrastructure side +- Ability to comfortably run on constrained environments by reducing application footprint + +Cons: + +- It is harder to implement GraphQL subscriptions as in case of Sidekiq as we need another way to pass subscriptions +- `api_v4` paths can be used in some services that are used by Sidekiq (e.g. `api_v4_projects_path`) +- url_helpers paths are used in models and services, that could be used by Sidekiq (e.g. `Gitlab::Routing.url_helpers.project_pipelines_path` is used by [ExpirePipelineCacheService](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/expire_pipeline_cache_service.rb#L20) in [ExpirePipelineCacheWorker](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/expire_pipeline_cache_worker.rb#L18)) + +#### Example: GraphQL + +[Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) + +- The [99% of changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=49c9881c6696eb620dccac71532a3173f5702ea8) as visible in the above MRs is moving files as-is. +- The [actual work](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=1d9a9edfa29ea6638e7d8a6712ddf09f5be77a44) on fixing cross-dependencies, specs, and configuring web_engine +- We [adapted](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=d7f862cc209ce242000b2aec88ff7f4485acdd92) CI to test `engines/web_engine/` as a self-sufficient component of stack + +Today, loading GraphQL requires a bunch of [dependencies](https://gitlab.com/gitlab-org/gitlab/-/issues/288044): + +> We also discovered that we load/require 14480 files, [memory-team-2gb-week#9](https://gitlab.com/gitlab-org/memory-team/memory-team-2gb-week/-/issues/9#note_452530513) +> when we start GitLab. 1274 files belong to GraphQL. This means that if we don't load 1274 application files +> and all related GraphQL gems when we don't need them (Sidekiq), we could save a lot of memory. + +GraphQL only needs to run in a specific context. If we could limit when it is being loaded we could effectively improve application efficiency, by reducing application load time and required memory. This, for example, is applicable for every size installation. + +A potential challenge with GraphQL and Websockets is that at some point we might want to use Action Cable subscriptions and push GraphQL/API payload from Sidekiq to clients. This would likely utilize Redis to pass data through. Where Sidekiq would publish information on Redis and ActionCable Node would pass through that information to connected clients. This way of working is possible in the above model, but we would have to use GraphQL or API (over HTTP endpoint) to calculate what should be sent. + +An alternative way is to use a notification system that would always make an `ActionCable` node (the one handling WebSockets) generate a payload based on a send query instead of performing passthrough. This could be applicable since `ActionCable` is the one handling a given connection for a client. This could have a downside of having to recalculate the same payload if many clients would be watching the same resource. However, this behavior of system might still be desired for security purposes, as generated payload might be dependent on permission of watching client (we would show different for anonymous, and different for the member of the project). + +#### Example: API + +[Draft: PoC - Move only Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982) + +- [99% of the changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982/diffs?commit_id=c8b72249b6e8f875ed4c713f0668207377604043), as visible in the above MRs, are moving the files as-is. +- The [actual work](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982/diffs?commit_id=00d9b54ba952c85ff4d158a18205c2fac13eaf8d) on fixing cross-dependencies, specs, configuring initializers, gems and routes. + +Grape::API is another example that only needs to run only in a web server context. + +Potential challenges with Grape API: + +- Currently there are some API::API dependencies in the models (e.g. `API::Helpers::Version` dependency in [project model](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/project.rb#L2019) or API::API dependency in GeoNode model for [`geo_retrieve_url`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/geo_node.rb#L183)) +- `api_v4` paths are used in helpers, presenters, and views (e.g. `api_v4_projects_path` in [PackagesHelper](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/helpers/packages_helper.rb#L17)) + +#### Example: Controllers + +[Draft: PoC - Move Controllers and Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720) + +- [99% of the changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720/diffs?commit_id=17174495cf3263c8e69a0420092d9fa759170aa6), as visible in the above MRs, are moving files as-is. +- The [actual work](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720/diffs?commit_id=39cc4bb1e0ce47f66605d06eb1b0d6b89ba174e6) on fixing cross-dependencies, specs, configuring initializers, gems and routes. + +Controllers, Serializers, some presenters and some of the Grape:Entities are also good examples that only need to be run in web server context. + +Potential challenges with moving Controllers: + +- We needed to extend `Gitlab::Patch::DrawRoute` in order to support `engines/web_engine/config/routes` and `engines/web_engine/ee/config/routes` in case when `web_engine` is loaded. Here is potential [solution](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720#note_506957398). +- `Gitlab::Routing.url_helpers` paths are used in models and services, that could be used by Sidekiq (e.g. `Gitlab::Routing.url_helpers.project_pipelines_path` is used by [ExpirePipelineCacheService](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/expire_pipeline_cache_service.rb#L20) in [ExpirePipelineCacheWorker](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/expire_pipeline_cache_worker.rb#L18))) + +### Packwerk + +NOTE: +Packwerk is currently accepting bug fixes only, and it is not being actively developed. Check for [more details](https://github.com/Shopify/packwerk#note-packwerk-is-considered-to-be-feature-complete-for-shopifys-uses-we-are-currently-accepting-bug-fixes-only-and-it-is-not-being-actively-developed-please-fork-this-project-if-you-are-interested-in-adding-new-features) + +## Future impact + +**Application Layers** and this proposal currently defines only `web_engine`. Following the same pattern we could easily introduce +additional engines dedicated for supporting that would allow us to maintain much better separation, lower memory usage +and much better maintainability of GitLab Rails into the future. + +This would be a framework for introducing all new interfaces for features that do not need to be part of the core codebase, +like support for additional Package services. Allowing us to better scale application in the future, but retaining a single codebase +and monolithic architecture of GitLab. + +As of today, it seems reasonable to define three **application layers**: + +- `gitlab-core`: a core functionality: DB structure, models, services, common library. It models a data access layer, and initially all services needed to run GitLab. This might be potentially be split in the future into smaller aspects +- `gitlab-web`: a Controllers/API/GraphQL/ActionCable functionality needed to run in a web server context (depends on `gitlab-core`) +- `gitlab-sidekiq`: a background jobs functionality needed to run Sidekiq Workers (depends on `gitlab-core`) + +This model is best described today as a shared core with satellite. The shared core defines data access layer, where as satellites define a way to present and process this data. Satellites can only talk with Core. They cannot directly load or talk to another satellite unless they use a well defined interface in form of API, GraphQL or Redis (as for scheduling Sidekiq jobs). + +It is reasonable to assume that we limit how many `engines` we allow. Initial proposal is to allow up to 5 engines +to be created to ensure that we do not have explosion of engines. + +## Issues and Merge Requests + +- [Split application into functional parts to ensure that only needed code is loaded with all dependencies](https://gitlab.com/gitlab-org/gitlab/-/issues/290935) +- [Provide mechanism to load GraphQL with all dependencies only when needed](https://gitlab.com/gitlab-org/gitlab/-/issues/288044) +- [Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) +- [Draft: PoC - Move Controllers and Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720) +- [Draft: PoC - Move only Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982) +- [Measure performance impact for proposed web_engine](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) +- [Create new models / classes within a module / namespace](https://gitlab.com/gitlab-org/gitlab/-/issues/212156) +- [Make teams to be maintainers of their code](https://gitlab.com/gitlab-org/gitlab/-/issues/25872) +- [Use nested structure to organize CI classes](https://gitlab.com/gitlab-org/gitlab/-/issues/209745) +- [WIP: Make it simple to build and use "Decoupled Services"](https://gitlab.com/gitlab-org/gitlab/-/issues/31121) +- [Rails takes awhile to boot, let's see if we can improve this](https://gitlab.com/gitlab-org/gitlab/-/issues/213992) + +## Who + +Proposal: + + + +| Role | Who +|------------------------------|-------------------------| +| Author | Kamil Trzciński | +| Architecture Evolution Coach | ? | +| Engineering Leader | ? | + +DRIs: + +| Role | Who +|------------------------------|------------------------| +| Product | ? | +| Leadership | Craig Gomes | +| Engineering | ? | + +Domain Experts: + +| Role | Who +|------------------------------|------------------------| +| Domain Expert | Nikola Milojevic | +| Domain Expert | ? | +| Domain Expert | ? | + + diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 86628b31536..403a1a1130a 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -1,6 +1,6 @@ --- -stage: package -group: package +stage: Package +group: Package info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers comments: false description: 'Container Registry metadata database' @@ -122,7 +122,7 @@ For similar reasons as highlighted above, currently, it's not feasible to extrac #### Additional Features -Due to the metadata limitations, it's currently not feasible to implement valuable features such as [pagination](https://gitlab.com/gitlab-org/container-registry/-/issues/13#note_271769891), filtering and sorting for HTTP API, and more advanced features such as the ability to [distinguish between Docker and Helm charts images](https://gitlab.com/gitlab-org/gitlab/issues/38047). +Due to the metadata limitations, it's currently not feasible to implement valuable features such as [pagination](https://gitlab.com/gitlab-org/container-registry/-/issues/13#note_271769891), filtering and sorting for HTTP API, and more advanced features such as the ability to [distinguish between Docker and Helm charts images](https://gitlab.com/gitlab-org/gitlab/-/issues/38047). Because of all these constraints, we decided to [freeze the development of new features](https://gitlab.com/gitlab-org/container-registry/-/issues/44) until we have a solution in place to overcome all these foundational limitations. diff --git a/doc/ci/README.md b/doc/ci/README.md index 30a6668dbfd..dc4312250ca 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false description: "Learn how to use GitLab CI/CD, the GitLab built-in Continuous Integration, Continuous Deployment, and Continuous Delivery toolset to build, test, and deploy your application." @@ -70,7 +70,7 @@ GitLab CI/CD supports numerous configuration options: | Configuration | Description | |:----------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------| | [Schedule pipelines](pipelines/schedules.md) | Schedule pipelines to run as often as you need. | -| [Custom path for `.gitlab-ci.yml`](pipelines/settings.md#custom-cicd-configuration-path) | Define a custom path for the CI/CD configuration file. | +| [Custom path for `.gitlab-ci.yml`](pipelines/settings.md#custom-cicd-configuration-file) | Define a custom path for the CI/CD configuration file. | | [Git submodules for CI/CD](git_submodules.md) | Configure jobs for using Git submodules. | | [SSH keys for CI/CD](ssh_keys/index.md) | Using SSH keys in your CI pipelines. | | [Pipeline triggers](triggers/README.md) | Trigger pipelines through the API. | diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index f2cb9500b2c..0778d598d32 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -1,77 +1,39 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: index, concepts, howto --- -# Cache dependencies in GitLab CI/CD +# Caching in GitLab CI/CD -GitLab CI/CD provides a caching mechanism that can be used to save time -when your jobs are running. +A cache is one or more files that a job downloads and saves. Subsequent jobs that use +the same cache don't have to download the files again, so they execute more quickly. -Caching is about speeding the time a job is executed by reusing the same -content of a previous job. Use caching when you are -developing software that depends on other libraries which are fetched via the -internet during build time. +To learn how to define the cache in your `.gitlab-ci.yml` file, +see the [`cache` reference](../yaml/README.md#cache). -If caching is enabled, it's shared between pipelines and jobs at the project -level by default. Caches are not shared across projects. +## How cache is different from artifacts -Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn -how it is defined in `.gitlab-ci.yml`. +Use cache for dependencies, like packages you download from the internet. +Cache is stored where GitLab Runner is installed and uploaded to S3 if +[distributed cache is enabled](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching). -## Cache vs artifacts +- You can define it per job by using the `cache:` keyword. Otherwise it is disabled. +- You can define it per job so that: + - Subsequent pipelines can use it. + - Subsequent jobs in the same pipeline can use it, if the dependencies are identical. +- You cannot share it between projects. -If you use cache and artifacts to store the same path in your jobs, the cache might -be overwritten because caches are restored before artifacts. - -Don't use caching for passing artifacts between stages, as it is designed to store -runtime dependencies needed to compile the project: - -- `cache`: **For storing project dependencies** - - Caches can increase the speed of a given job in subsequent pipelines. You can - store downloaded dependencies so that they don't have to be fetched from the - internet again. Dependencies include things like npm packages, Go vendor packages, and so on. - You can configure a cache to pass intermediate build results between stages, - but you should use artifacts instead. - -- `artifacts`: **Use for stage results that are passed between stages.** - - Artifacts are files that are generated by a job so they can be stored and uploaded. You can - fetch and use artifacts in jobs in later stages of the same pipeline. You can't - create an artifact in a job in one stage, and use this artifact in a different job in - the same stage. This data is not available in different pipelines, but can be downloaded - from the UI. - - If you download modules while building your application, you can declare them as - artifacts and subsequent stage jobs can use them. +Use artifacts to pass intermediate build results between stages. +Artifacts are generated by a job, stored in GitLab, and can be downloaded. - You can define an [expiry time](../yaml/README.md#artifactsexpire_in) so artifacts - are deleted after a defined time. Use [dependencies](../yaml/README.md#dependencies) - to control which jobs fetch the artifacts. +- You can define artifacts per job. Subsequent jobs in later stages of the same + pipeline can use them. +- You can't use the artifacts in a different pipeline. - Artifacts can also be used to make files available for download after a pipeline - completes, like a build image. - -Caches: - -- Are disabled if not defined globally or per job (using `cache:`). -- Are available for all jobs in your `.gitlab-ci.yml` if enabled globally. -- Can be used in subsequent pipelines by the same job in which the cache was created (if not defined globally). -- Are stored where GitLab Runner is installed **and** uploaded to S3 if [distributed cache is enabled](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching). -- If defined per job, are used: - - By the same job in a subsequent pipeline. - - By subsequent jobs in the same pipeline, if they have identical dependencies. - -Artifacts: - -- Are disabled if not defined per job (using `artifacts:`). -- Can only be enabled per job, not globally. -- Are created during a pipeline and can be used by subsequent jobs in the same pipeline. -- Are always uploaded to GitLab (known as coordinator). -- Can have an expiration value for controlling disk usage (30 days by default). +Artifacts expire after 30 days unless you define an [expiration time](../yaml/README.md#artifactsexpire_in). +Use [dependencies](../yaml/README.md#dependencies) to control which jobs fetch the artifacts. Both artifacts and caches define their paths relative to the project directory, and can't link to files outside it. @@ -81,10 +43,9 @@ can't link to files outside it. To ensure maximum availability of the cache, when you declare `cache` in your jobs, use one or more of the following: -- [Tag your runners](../runners/README.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) and use the tag on jobs +- [Tag your runners](../runners/configure_runners.md#use-tags-to-limit-the-number-of-jobs-using-the-runner) and use the tag on jobs that share their cache. -- [Use sticky runners](../runners/README.md#prevent-a-specific-runner-from-being-enabled-for-other-projects) - that are only available to a particular project. +- [Use runners that are only available to a particular project](../runners/runners_scope.md#prevent-a-specific-runner-from-being-enabled-for-other-projects). - [Use a `key`](../yaml/README.md#cachekey) that fits your workflow (for example, different caches on each branch). For that, you can take advantage of the [predefined CI/CD variables](../variables/README.md#predefined-cicd-variables). @@ -102,7 +63,7 @@ For runners to work with caches efficiently, you must do one of the following: Read about the [availability of the cache](#availability-of-the-cache) to learn more about the internals and get a better idea how cache works. -### Share caches across the same branch +### Share caches between jobs in the same branch Define a cache with the `key: ${CI_COMMIT_REF_SLUG}` so that jobs of each branch always use the same cache: @@ -130,7 +91,7 @@ cache: key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" ``` -### Share caches across different branches +### Share caches across jobs in different branches To share a cache across all branches and all jobs, use the same key for everything: @@ -146,7 +107,7 @@ cache: key: ${CI_JOB_NAME} ``` -### Disable cache on specific jobs +### Disable cache for specific jobs If you have defined the cache globally, it means that each job uses the same definition. You can override this behavior per-job, and if you want to @@ -189,7 +150,7 @@ The most common use case of caching is to avoid downloading content like depende or libraries repeatedly between subsequent runs of jobs. Node.js packages, PHP packages, Ruby gems, Python libraries, and others can all be cached. -For more examples, check out our [GitLab CI/CD templates](https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates). +For more examples, check out our [GitLab CI/CD templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). ### Cache Node.js dependencies @@ -201,7 +162,7 @@ Instead, we tell npm to use `./.npm`, and cache it per-branch: ```yaml # -# https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml # image: node:latest @@ -219,7 +180,7 @@ test_async: - node ./specs/start.js ./specs/async.spec.js ``` -### Caching PHP dependencies +### Cache PHP dependencies Assuming your project is using [Composer](https://getcomposer.org/) to install the PHP dependencies, the following example defines `cache` globally so that @@ -228,7 +189,7 @@ are cached per-branch: ```yaml # -# https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/PHP.gitlab-ci.yml # image: php:7.2 @@ -248,7 +209,7 @@ test: - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never ``` -### Caching Python dependencies +### Cache Python dependencies Assuming your project is using [pip](https://pip.pypa.io/en/stable/) to install the Python dependencies, the following example defines `cache` globally so that @@ -257,7 +218,7 @@ pip's cache is defined under `.cache/pip/` and both are cached per-branch: ```yaml # -# https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml # image: python:latest @@ -289,7 +250,7 @@ test: - flake8 . ``` -### Caching Ruby dependencies +### Cache Ruby dependencies Assuming your project is using [Bundler](https://bundler.io) to install the gem dependencies, the following example defines `cache` globally so that all @@ -297,7 +258,7 @@ jobs inherit it. Gems are installed in `vendor/ruby/` and are cached per-branch: ```yaml # -# https://gitlab.com/gitlab-org/gitlab/tree/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml # image: ruby:2.6 @@ -347,7 +308,7 @@ deploy_job: - bundle exec deploy ``` -### Caching Go dependencies +### Cache Go dependencies Assuming your project is using [Go Modules](https://github.com/golang/go/wiki/Modules) to install Go dependencies, the following example defines `cache` in a `go-cache` template, that @@ -396,6 +357,9 @@ machine where the runner is installed and depends on the type of the executor. | [Docker](https://docs.gitlab.com/runner/executors/docker.html) | Locally, stored under [Docker volumes](https://docs.gitlab.com/runner/executors/docker.html#the-builds-and-cache-storage): `/var/lib/docker/volumes//_data////cache.zip`. | | [Docker machine](https://docs.gitlab.com/runner/executors/docker_machine.html) (autoscale runners) | Behaves the same as the Docker executor. | +If you use cache and artifacts to store the same path in your jobs, the cache might +be overwritten because caches are restored before artifacts. + ### How archiving and extracting works This example has two jobs that belong to two consecutive stages: diff --git a/doc/ci/chatops/README.md b/doc/ci/chatops/README.md index c94d6e3ea80..577a80407d7 100644 --- a/doc/ci/chatops/README.md +++ b/doc/ci/chatops/README.md @@ -1,5 +1,6 @@ --- redirect_to: 'index.md' +remove_date: '2021-05-01' --- This document was moved to [another location](index.md). diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index 38930eb60ad..4d3f12dff05 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto --- diff --git a/doc/ci/ci_cd_for_external_repos/github_integration.md b/doc/ci/ci_cd_for_external_repos/github_integration.md index deadc4458a2..c6fa049dc6e 100644 --- a/doc/ci/ci_cd_for_external_repos/github_integration.md +++ b/doc/ci/ci_cd_for_external_repos/github_integration.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto --- diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index cc6c629fb47..8c961ea6128 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: index, howto --- diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index dab9d8b78ae..e9725a29fc7 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference --- diff --git a/doc/ci/docker/README.md b/doc/ci/docker/README.md index c94d6e3ea80..577a80407d7 100644 --- a/doc/ci/docker/README.md +++ b/doc/ci/docker/README.md @@ -1,5 +1,6 @@ --- redirect_to: 'index.md' +remove_date: '2021-05-01' --- This document was moved to [another location](index.md). diff --git a/doc/ci/docker/index.md b/doc/ci/docker/index.md index 0897bb183e5..20599c5ca85 100644 --- a/doc/ci/docker/index.md +++ b/doc/ci/docker/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false type: index diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 90a33478239..9dac08324c8 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: concepts, howto --- @@ -25,9 +25,8 @@ To enable Docker commands for your CI/CD jobs, you can use: If you don't want to execute a runner in privileged mode, but want to use `docker build`, you can also [use kaniko](using_kaniko.md). -If you are using shared runners on GitLab.com, see -[GitLab.com shared runners](../../user/gitlab_com/index.md#shared-runners) -to learn more about how these runners are configured. +If you are using shared runners on GitLab.com, +[learn more about how these runners are configured](../runners/README.md). ### Use the shell executor @@ -91,7 +90,7 @@ The Docker image has all of the `docker` tools installed and can run the job script in context of the image in privileged mode. We recommend you use [Docker-in-Docker with TLS enabled](#docker-in-docker-with-tls-enabled), -which is supported by [GitLab.com shared runners](../../user/gitlab_com/index.md#shared-runners). +which is supported by [GitLab.com shared runners](../runners/README.md). You should always specify a specific version of the image, like `docker:19.03.12`. If you use a tag like `docker:stable`, you have no control over which version is used. diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 173701ef358..29f4053f720 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -128,6 +128,10 @@ For example, the following two definitions are equal: - name: redis:latest ``` +## Where scripts are executed + +When a CI job runs in a Docker container, the `before_script`, `script`, and `after_script` commands run in the `/builds//` directory. Your image may have a different default `WORKDIR` defined. To move to your `WORKDIR`, save the `WORKDIR` as an environment variable so you can reference it in the container during the job's runtime. + ### Available settings for `image` > Introduced in GitLab and GitLab Runner 9.4. diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 0344e736dd4..05769cc8f75 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto --- @@ -99,8 +99,8 @@ build: KANIKOCFG="${KANIKOCFG} }" echo "${KANIKOCFG}" > /kaniko/.docker/config.json - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile $KANIKOPROXYBUILDARGS --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG - only: - - tags + rules: + - if: $CI_COMMIT_TAG ``` ## Using a registry with a custom certificate @@ -133,7 +133,7 @@ The [Least Privilege Container Builds with Kaniko on GitLab](https://www.youtube video is a walkthrough of the [Kaniko Docker Build](https://gitlab.com/guided-explorations/containers/kaniko-docker-build) Guided Exploration project pipeline. It was tested on: -- [GitLab.com shared runners](../../user/gitlab_com/index.md#shared-runners) +- [GitLab.com shared runners](../runners/README.md) - [The Kubernetes runner executor](https://docs.gitlab.com/runner/executors/kubernetes.html) The example can be copied to your own group or instance for testing. More details diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index 72fd9833df1..4633cc1a3f8 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Continuous Integration +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto --- diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index 6fda6bb0d8b..c57dc99f341 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -117,7 +117,7 @@ The other pipelines don't get the protected variable. You can also [scope variables to specific environments](../variables/where_variables_can_be_used.md#variables-with-an-environment-scope). We recommend that you use protected variables on protected environments to make sure that the secrets aren't exposed unintentionally. You can also define production secrets on the -[runner side](../runners/README.md#prevent-runners-from-revealing-sensitive-information). +[runner side](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). This prevents other maintainers from reading the secrets and makes sure that the runner only runs on protected branches. @@ -141,7 +141,7 @@ reference a file in another project with a completely different set of permissio In this scenario, the `gitlab-ci.yml` is publicly accessible, but can only be edited by users with appropriate permissions in the other project. -For more information, see [Custom CI/CD configuration path](../pipelines/settings.md#custom-cicd-configuration-path). +For more information, see [Custom CI/CD configuration path](../pipelines/settings.md#custom-cicd-configuration-file). ## Troubleshooting diff --git a/doc/ci/environments/environments_dashboard.md b/doc/ci/environments/environments_dashboard.md index 4ee9aa9a5ba..a89bc1c89aa 100644 --- a/doc/ci/environments/environments_dashboard.md +++ b/doc/ci/environments/environments_dashboard.md @@ -20,8 +20,8 @@ see which pipelines are green and which are red allowing you to diagnose if there is a block at a particular point, or if there's a more systemic problem you need to investigate. -You can access the dashboard from the top bar by clicking -**More > Environments**. +You can access the dashboard on the top bar by selecting +**Menu > Environments**. ![Environments Dashboard with projects](img/environments_dashboard_v12_5.png) diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 06618a820db..62c58302886 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -31,7 +31,7 @@ Prerequisites: To view a list of environments and deployments: -1. Go to the project's **Operations > Environments** page. +1. Go to the project's **Deployments > Environments** page. The environments are displayed. ![Environments list](img/environments_list.png) @@ -57,7 +57,7 @@ You can create an environment and deployment in the UI or in your `.gitlab-ci.ym In the UI: -1. Go to the project's **Operations > Environments** page. +1. Go to the project's **Deployments > Environments** page. 1. Select **New environment**. 1. Enter a name and external URL. 1. Select **Save**. @@ -99,10 +99,10 @@ deploy_review: environment: name: review/$CI_COMMIT_REF_NAME url: https://$CI_ENVIRONMENT_SLUG.example.com - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - if: $CI_COMMIT_BRANCH ``` In this example: @@ -136,7 +136,7 @@ you can use tiers: | Environment tier | Environment name examples | |------------------|----------------------------------------------------| | `production` | Production, Live | -| `staging` | Staging, Model, Pre, Demo | +| `staging` | Staging, Model, Demo | | `testing` | Test, QC | | `development` | Dev, [Review apps](../review_apps/index.md), Trunk | | `other` | | @@ -158,8 +158,8 @@ deploy_prod: name: production url: https://example.com when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ``` The `when: manual` action: @@ -200,8 +200,8 @@ deploy: url: https://example.com kubernetes: namespace: production - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ``` When you use the GitLab Kubernetes integration to deploy to a Kubernetes cluster, @@ -326,7 +326,7 @@ If there is a problem with a deployment, you can retry it or roll it back. To retry or rollback a deployment: -1. Go to the project's **Operations > Environments**. +1. Go to the project's **Deployments > Environments**. 1. Select the environment. 1. To the right of the deployment name: - To retry a deployment, select **Re-deploy to environment**. @@ -409,7 +409,7 @@ The job with [`action: stop` might not run](#the-job-with-action-stop-doesnt-run if it's in a later stage than the job that started the environment. If you can't use [pipelines for merge requests](../merge_request_pipelines/index.md), -set the [`GIT_STRATEGY`](../runners/README.md#git-strategy) to `none` in the +set the [`GIT_STRATEGY`](../runners/configure_runners.md#git-strategy) to `none` in the `stop_review` job. Then the [runner](https://docs.gitlab.com/runner/) doesn't try to check out the code after the branch is deleted. @@ -465,7 +465,7 @@ GitLab automatically triggers the `stop_review_app` job to stop the environment. You can view a deployment's expiration date in the GitLab UI. -1. Go to the project's **Operations > Environments** page. +1. Go to the project's **Deployments > Environments** page. 1. Select the name of the deployment. In the top left, next to the environment name, the expiration date is displayed. @@ -474,7 +474,7 @@ In the top left, next to the environment name, the expiration date is displayed. You can manually override a deployment's expiration date. -1. Go to the project's **Operations > Environments** page. +1. Go to the project's **Deployments > Environments** page. 1. Select the deployment name. 1. On the top right, select the thumbtack (**{thumbtack}**). @@ -491,7 +491,7 @@ You can delete [stopped environments](#stopping-an-environment) in the GitLab UI To delete a stopped environment in the GitLab UI: -1. Go to the project's **Operations > Environments** page. +1. Go to the project's **Deployments > Environments** page. 1. Select the **Stopped** tab. 1. Next to the environment you want to delete, select **Delete environment**. 1. On the confirmation dialog box, select **Delete environment**. diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index df0bb2817ab..c276059cb9e 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -23,8 +23,8 @@ with the right privileges can deploy to it, thus keeping it safe. NOTE: A GitLab admin is always allowed to use environments, even if they are protected. -To protect, update, or unprotect an environment, you need to have at least -[Maintainer permissions](../../user/permissions.md). +To protect, update, or unprotect an environment, you need to have at least the +[Maintainer role](../../user/permissions.md). ## Protecting environments @@ -79,7 +79,8 @@ Alternatively, you can use the API to protect an environment: 1. Use the API to add a user to the group as a reporter: ```shell - $ curl --request POST --header "PRIVATE-TOKEN: " --data "user_id=3222377&access_level=20" "https://gitlab.com/api/v4/groups/9899826/members" + $ curl --request POST --header "PRIVATE-TOKEN: " \ + --data "user_id=3222377&access_level=20" "https://gitlab.com/api/v4/groups/9899826/members" {"id":3222377,"name":"Sean Carroll","username":"sfcarroll","state":"active","avatar_url":"https://assets.gitlab-static.net/uploads/-/system/user/avatar/3222377/avatar.png","web_url":"https://gitlab.com/sfcarroll","access_level":20,"created_at":"2020-10-26T17:37:50.309Z","expires_at":null} ``` @@ -87,7 +88,8 @@ Alternatively, you can use the API to protect an environment: 1. Use the API to add the group to the project as a reporter: ```shell - $ curl --request POST --header "PRIVATE-TOKEN: " --request POST "https://gitlab.com/api/v4/projects/22034114/share?group_id=9899826&group_access=20" + $ curl --request POST --header "PRIVATE-TOKEN: " \ + --request POST "https://gitlab.com/api/v4/projects/22034114/share?group_id=9899826&group_access=20" {"id":1233335,"project_id":22034114,"group_id":9899826,"group_access":20,"expires_at":null} ``` @@ -95,7 +97,8 @@ Alternatively, you can use the API to protect an environment: 1. Use the API to add the group with protected environment access: ```shell - curl --header 'Content-Type: application/json' --request POST --data '{"name": "production", "deploy_access_levels": [{"group_id": 9899826}]}' --header "PRIVATE-TOKEN: " "https://gitlab.com/api/v4/projects/22034114/protected_environments" + curl --header 'Content-Type: application/json' --request POST --data '{"name": "production", "deploy_access_levels": [{"group_id": 9899826}]}' \ + --header "PRIVATE-TOKEN: " "https://gitlab.com/api/v4/projects/22034114/protected_environments" ``` The group now has access and can be seen in the UI. @@ -151,6 +154,129 @@ be re-entered if the environment is re-protected. For more information, see [Deployment safety](deployment_safety.md). +## Group-level protected environments + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215888) in [GitLab Premium](https://about.gitlab.com/pricing/) 14.0. +> - [Deployed behind a feature flag](../../user/feature_flags.md), disabled by default. +> - Disabled on GitLab.com. +> - Not recommended for production use. +> - To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-group-level-protected-environments). **(FREE SELF)** + +This in-development feature might not be available for your use. There can be +[risks when enabling features still in development](../../user/feature_flags.md#risks-when-enabling-features-still-in-development). +Refer to this feature's version history for more details. + +Typically, large enterprise organizations have an explicit permission boundary +between [developers and operators](https://about.gitlab.com/topics/devops/). +Developers build and test their code, and operators deploy and monitor the +application. With group-level protected environments, the permission of each +group is carefully configured in order to prevent unauthorized access and +maintain proper separation of duty. Group-level protected environments +extend the [project-level protected environments](#protecting-environments) +to the group-level. + +The permissions of deployments can be illustrated in the following table: + +| Environment | Developer | Operator | Category | +|-------------|------------|----------|----------| +| Development | Allowed | Allowed | Lower environment | +| Testing | Allowed | Allowed | Lower environment | +| Staging | Disallowed | Allowed | Higher environment | +| Production | Disallowed | Allowed | Higher environment | + +_(Reference: [Deployment environments on Wikipedia](https://en.wikipedia.org/wiki/Deployment_environment))_ + +### Group-level protected environments names + +Contrary to project-level protected environments, group-level protected +environments use the [deployment tier](index.md#deployment-tier-of-environments) +as their name. + +A group may consist of many project environments that have unique names. +For example, Project-A has a `gprd` environment and Project-B has a `Production` +environment, so protecting a specific environment name doesn't scale well. +By using deployment tiers, both are recognized as `production` deployment tier +and are protected at the same time. + +### Configure group-level memberships + +In an enterprise organization, with thousands of projects under a single group, +ensuring that all of the [project-level protected environments](#protecting-environments) +are properly configured is not a scalable solution. For example, a developer +might gain privileged access to a higher environment when they are added as a +maintainer to a new project. Group-level protected environments can be a solution +in this situation. + +To maximize the effectiveness of group-level protected environments, +[group-level memberships](../../user/group/index.md) must be correctly +configured: + +- Operators should be assigned the [maintainer role](../../user/permissions.md) + (or above) to the top-level group. They can maintain CI/CD configurations for + the higher environments (such as production) in the group-level settings page, + wnich includes group-level protected environments, + [group-level runners](../runners/runners_scope.md#group-runners), + [group-level clusters](../../user/group/clusters/index.md), etc. Those + configurations are inherited to the child projects as read-only entries. + This ensures that only operators can configure the organization-wide + deployment ruleset. +- Developers should be assigned the [developer role](../../user/permissions.md) + (or below) at the top-level group, or explicitly assigned to a child project + as maintainers. They do *NOT* have access to the CI/CD configurations in the + top-level group, so operators can ensure that the critical configuration won't + be accidentally changed by the developers. +- For sub-groups and child projects: + - Regarding [sub-groups](../../user/group/subgroups/index.md), if a higher + group has configured the group-level protected environment, the lower groups + cannot override it. + - [Project-level protected environments](#protecting-environments) can be + combined with the group-level setting. If both group-level and project-level + environment configurations exist, the user must be allowed in **both** + rulesets in order to run a deployment job. + - Within a project or a sub-group of the top-level group, developers can be + safely assigned the Maintainer role to tune their lower environments (such + as `testing`). + +Having this configuration in place: + +- If a user is about to run a deployment job in a project and allowed to deploy + to the environment, the deployment job proceeds. +- If a user is about to run a deployment job in a project but disallowed to + deploy to the environment, the deployment job fails with an error message. + +### Protect a group-level environment + +To protect a group-level environment: + +1. Make sure your environments have the correct + [`deployment_tier`](index.md#deployment-tier-of-environments) defined in + `gitlab-ci.yml`. +1. Configure the group-level protected environments via the + [REST API](../../api/group_protected_environments.md). + +NOTE: +Configuration [via the UI](https://gitlab.com/gitlab-org/gitlab/-/issues/325249) +is scheduled for a later release. + +### Enable or disable Group-level protected environments **(FREE SELF)** + +Group-level protected environments is under development and not ready for production use. It is +deployed behind a feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:group_level_protected_environments) +``` + +To disable it: + +```ruby +Feature.disable(:group_level_protected_environments) +``` + @@ -318,7 +318,7 @@ Markdown code embeds the test coverage report badge of the `coverage` job into your `README.md`: ```markdown -![coverage](https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=coverage) +![coverage](https://gitlab.com/gitlab-org/gitlab/badges/main/coverage.svg?job=coverage) ``` ### Badge styles @@ -331,7 +331,7 @@ Pipeline badges can be rendered in different styles by adding the `style=style_n https://gitlab.example.com///badges//coverage.svg?style=flat ``` - ![Badge flat style](https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=coverage&style=flat) + ![Badge flat style](https://gitlab.com/gitlab-org/gitlab/badges/main/coverage.svg?job=coverage&style=flat) - Flat square ([Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30120) in GitLab 11.8): @@ -339,7 +339,7 @@ Pipeline badges can be rendered in different styles by adding the `style=style_n https://gitlab.example.com///badges//coverage.svg?style=flat-square ``` - ![Badge flat square style](https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=coverage&style=flat-square) + ![Badge flat square style](https://gitlab.com/gitlab-org/gitlab/badges/main/coverage.svg?job=coverage&style=flat-square) ### Custom badge text @@ -348,10 +348,10 @@ Pipeline badges can be rendered in different styles by adding the `style=style_n The text for a badge can be customized to differentiate between multiple coverage jobs that run in the same pipeline. Customize the badge text and width by adding the `key_text=custom_text` and `key_width=custom_key_width` parameters to the URL: ```plaintext -https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=130 +https://gitlab.com/gitlab-org/gitlab/badges/main/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=130 ``` -![Badge with custom text and width](https://gitlab.com/gitlab-org/gitlab/badges/master/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=130) +![Badge with custom text and width](https://gitlab.com/gitlab-org/gitlab/badges/main/coverage.svg?job=karma&key_text=Frontend+Coverage&key_width=130) - -| Usage | Guidance | -|-----------------------|----------| -| above | Try to avoid extra words when referring to an example or table in a documentation page, but if required, use **previously** instead. | -| admin, admin area | Use **administration**, **administrator**, **administer**, or **Admin Area** instead. ([Vale](../testing.md#vale) rule: [`Admin.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Admin.yml)) | -| allow, enable | Try to avoid, unless you are talking about security-related features. For example, instead of "This feature allows you to create a pipeline," use "Use this feature to create a pipeline." This phrasing is more active and is from the user perspective, rather than the person who implemented the feature. [View details](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/allow-allows). | -| and/or | Use **or** instead, or another sensible construction. | -| below | Try to avoid extra words when referring to an example or table in a documentation page, but if required, use **following** instead. | -| currently | Do not use when talking about the product or its features. The documentation describes the product as it is today. ([Vale](../testing.md#vale) rule: [`CurrentStatus.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/CurrentStatus.yml)) | -| easily | Do not use. If the user doesn't find the process to be these things, we lose their trust. | -| e.g. | Do not use Latin abbreviations. Use **for example**, **such as**, **for instance**, or **like** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) | -| future tense | When possible, use present tense instead. For example, use `after you execute this command, GitLab displays the result` instead of `after you execute this command, GitLab will display the result`. ([Vale](../testing.md#vale) rule: [`FutureTense.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FutureTense.yml)) | -| handy | Do not use. If the user doesn't find the process to be these things, we lose their trust. | -| high availability, HA | Do not use. Instead, direct readers to the GitLab [reference architectures](../../../administration/reference_architectures/index.md) for information about configuring GitLab for handling greater amounts of users. | -| I | Do not use first-person singular. Use **you**, **we**, or **us** instead. ([Vale](../testing.md#vale) rule: [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml)) | -| i.e. | Do not use Latin abbreviations. Use **that is** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) | -| jargon | Do not use. Define the term or [link to a definition](#links-to-external-documentation). | -| may, might | **Might** means something has the probability of occurring. **May** gives permission to do something. Consider **can** instead of **may**. | -| me, myself, mine | Do not use first-person singular. Use **you**, **we**, or **us** instead. ([Vale](../testing.md#vale) rule: [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml)) | -| please | Do not use. For details, see the [Microsoft style guide](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). | -| profanity | Do not use. Doing so may negatively affect other users and contributors, which is contrary to the GitLab value of [Diversity, Inclusion, and Belonging](https://about.gitlab.com/handbook/values/#diversity-inclusion). | -| scalability | Do not use when talking about increasing GitLab performance for additional users. The words scale or scaling are sometimes acceptable, but references to increasing GitLab performance for additional users should direct readers to the GitLab [reference architectures](../../../administration/reference_architectures/index.md) page. | -| simply | Do not use. If the user doesn't find the process to be these things, we lose their trust. | -| slashes | Instead of **and/or**, use **or** or another sensible construction. This rule also applies to other slashes, like **follow/unfollow**. Some exceptions (like **CI/CD**) are allowed. | -| subgroup | Use instead of `sub-group`. | -| that | Do not use. Example: `the file that you save` can be `the file you save`. | -| useful | Do not use. If the user doesn't find the process to be these things, we lose their trust. | -| utilize | Do not use. Use **use** instead. It's more succinct and easier for non-native English speakers to understand. | -| via | Do not use Latin abbreviations. Use **with**, **through**, or **by using** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) | - - ### Contractions Contractions are encouraged, and can create a friendly and informal tone, @@ -540,7 +501,7 @@ especially in tutorials, instructional documentation, and Some contractions, however, should be avoided: -- Do not use [the word GitLab in a contraction](#trademark). +- Do not use the word "GitLab" in a contraction. - Do not use contractions with a proper noun and a verb. For example: @@ -1108,36 +1069,42 @@ document to ensure it links to the most recent version of the file. ## Navigation -When documenting navigation through the user interface: - -- Use the exact wording as shown in the UI, including any capital letters as-is. -- Use bold text for navigation items. +When documenting navigation through the user interface, use these terms and styles. ### What to call the menus Use these terms when referring to the main GitLab user interface elements: -- **Top menu**: This is the top menu that spans the width of the user interface. - It includes the GitLab logo, search field, counters, and the user's avatar. +- **Top bar**: This is the top bar that spans the width of the user interface. + It includes the menu, the GitLab logo, search field, counters, and the user's avatar. - **Left sidebar**: This is the navigation sidebar on the left of the user interface, specific to the project or group. - **Right sidebar**: This is the navigation sidebar on the right of the user interface, specific to the open issue, merge request, or epic. -### How to document the left sidebar +### How to document the menus -To be consistent, use this format when you refer to the left sidebar. +To be consistent, use this format when you write about UI navigation. -- Go to your project and select **Settings > CI/CD**. -- Go to your group and select **Settings > CI/CD**. -- Go to the Admin Area (**{admin}**) and select **Overview > Projects**. +1. On the top bar, select **Menu > Project** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **General pipelines**. -For expandable menus, use this format: +Another example: -1. Go to your group and select **Settings > CI/CD**. +1. On the top bar, select **Menu > Group** and find your group. +1. On the left sidebar, select **Settings > CI/CD**. 1. Expand **General pipelines**. +An Admin Area example: + +`1. On the top bar, select **Menu >** **{admin}** **Admin**.` + +This text generates this HTML: + +1. On the top bar, select **Menu >** **{admin}** **Admin**. + ## Images Images, including screenshots, can help a reader better understand a concept. @@ -1309,7 +1276,7 @@ hidden on the documentation site, but is displayed by `/help`. ## Code blocks - Always wrap code added to a sentence in inline code blocks (`` ` ``). - For example, `.gitlab-ci.yml`, `git add .`, `CODEOWNERS`, or `only: [master]`. + For example, `.gitlab-ci.yml`, `git add .`, `CODEOWNERS`, or `only: [main]`. File names, commands, entries, and anything that refers to code should be added to code blocks. To make things easier for the user, always add a full code block for things that can be useful to copy and paste, as they can do it @@ -1416,10 +1383,10 @@ readability of the text. For example, this Markdown adds little to the accompanying text: ```markdown -1. Go to **{home}** **Project overview > Details**. +1. Go to **{home}** **Project information > Details**. ``` -1. Go to **{home}** **Project overview > Details**. +1. Go to **{home}** **Project information > Details**. However, these tables might help the reader connect the text to the user interface: diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md new file mode 100644 index 00000000000..fd8766bbfb6 --- /dev/null +++ b/doc/development/documentation/styleguide/word_list.md @@ -0,0 +1,163 @@ +--- +stage: none +group: Style Guide +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +description: 'Writing styles, markup, formatting, and other standards for GitLab Documentation.' +--- + +# A-Z word list + +To help ensure consistency in the documentation, follow this guidance. + + + + +## above + +Try to avoid extra words when referring to an example or table in a documentation page, but if required, use **previously** instead. + +## admin, admin area + +Use **administration**, **administrator**, **administer**, or **Admin Area** instead. ([Vale](../testing.md#vale) rule: [`Admin.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Admin.yml)) + +## allow, enable + +Try to avoid, unless you are talking about security-related features. For example, instead of "This feature allows you to create a pipeline," use "Use this feature to create a pipeline." This phrasing is more active and is from the user perspective, rather than the person who implemented the feature. [View details](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/allow-allows). + +## and/or + +Instead of **and/or**, use or or rewrite the sentence to spell out both options. + +## below + +Try to avoid extra words when referring to an example or table in a documentation page, but if required, use **following** instead. + +## currently + +Do not use when talking about the product or its features. The documentation describes the product as it is today. ([Vale](../testing.md#vale) rule: [`CurrentStatus.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/CurrentStatus.yml)) + +## Developer + +When writing about the Developer role, use a capital "D." Do not use the phrase, "if you are a developer" +to mean someone who is assigned the Developer role. Instead, write it out. "If you are assigned the Developer role..." + +Do not use "Developer permissions." A user who is assigned the Developer role has a set of associated permissions. + +## easily + +Do not use. If the user doesn't find the process to be these things, we lose their trust. + +## e.g. + +Do not use Latin abbreviations. Use **for example**, **such as**, **for instance**, or **like** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) + +## future tense + +When possible, use present tense instead. For example, use `after you execute this command, GitLab displays the result` instead of `after you execute this command, GitLab will display the result`. ([Vale](../testing.md#vale) rule: [`FutureTense.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FutureTense.yml)) + +## GitLab + +Do not make possessive (GitLab's). This guidance follows [GitLab Brand Guidelines](https://about.gitlab.com/handbook/marketing/corporate-marketing/brand-activation/brand-guidelines/#trademark). + +## Guest + +When writing about the Guest role, use a capital "G." Do not use the phrase, "if you are a guest" +to mean someone who is assigned the Guest role. Instead, write it out. "If you are assigned the Guest role..." + +Do not use "Guest permissions." A user who is assigned the Guest role has a set of associated permissions. + +## handy + +Do not use. If the user doesn't find the process to be these things, we lose their trust. + +## high availability, HA + +Do not use. Instead, direct readers to the GitLab [reference architectures](../../../administration/reference_architectures/index.md) for information about configuring GitLab for handling greater amounts of users. + +## I + +Do not use first-person singular. Use **you**, **we**, or **us** instead. ([Vale](../testing.md#vale) rule: [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml)) + +## i.e. + +Do not use Latin abbreviations. Use **that is** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) + +## Maintainer + +When writing about the Maintainer role, use a capital "M." Do not use the phrase, "if you are a maintainer" +to mean someone who is assigned the Maintainer role. Instead, write it out. "If you are assigned the Maintainer role..." + +Do not use "Maintainer permissions." A user who is assigned the Maintainer role has a set of associated permissions. + +## may, might + +**Might** means something has the probability of occurring. **May** gives permission to do something. Consider **can** instead of **may**. + +## me, myself, mine + +Do not use first-person singular. Use **you**, **we**, or **us** instead. ([Vale](../testing.md#vale) rule: [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml)) + +## Owner + +When writing about the Owner role, use a capital "M." Do not use the phrase, "if you are an owner" +to mean someone who is assigned the Owner role. Instead, write it out. "If you are assigned the Owner role..." + +Do not use "Owner permissions." A user who is assigned the Owner role has a set of associated permissions. + +## permissions + +Do not use roles and permissions interchangeably. Each user is assigned a role. Each role includes a set of permissions. + +## please + +Do not use. For details, see the [Microsoft style guide](https://docs.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). + +## profanity + +Do not use. Doing so may negatively affect other users and contributors, which is contrary to the GitLab value of [Diversity, Inclusion, and Belonging](https://about.gitlab.com/handbook/values/#diversity-inclusion). + +## Reporter + +When writing about the Reporter role, use a capital "R." Do not use the phrase, "if you are a reporter" +to mean someone who is assigned the Reporter role. Instead, write it out. "If you are assigned the Reporter role..." + +Do not use "Reporter permissions." A user who is assigned the Reporter role has a set of associated permissions. + +## roles + +Do not use roles and permissions interchangeably. Each user is assigned a role. Each role includes a set of permissions. + +## scalability + +Do not use when talking about increasing GitLab performance for additional users. The words scale or scaling are sometimes acceptable, but references to increasing GitLab performance for additional users should direct readers to the GitLab [reference architectures](../../../administration/reference_architectures/index.md) page. + +## simply + +Do not use. If the user doesn't find the process to be these things, we lose their trust. + +## slashes + +Instead of **and/or**, use **or** or another sensible construction. This rule also applies to other slashes, like **follow/unfollow**. Some exceptions (like **CI/CD**) are allowed. + +## subgroup + +Use instead of `sub-group`. + +## that + +Do not use. Example: `the file that you save` can be `the file you save`. + +## useful + +Do not use. If the user doesn't find the process to be these things, we lose their trust. + +## utilize + +Do not use. Use **use** instead. It's more succinct and easier for non-native English speakers to understand. + +## via + +Do not use Latin abbreviations. Use **with**, **through**, or **by using** instead. ([Vale](../testing.md#vale) rule: [`LatinTerms.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/LatinTerms.yml)) + + + diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index af95f3b9023..b634e2b93db 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -20,7 +20,7 @@ For the specifics of each test run in our CI/CD pipelines, see the configuration in the relevant projects: - -- +- - - @@ -44,7 +44,7 @@ To run tests locally, it's important to: ### Lint checks -Lint checks are performed by the [`lint-doc.sh`](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/lint-doc.sh) +Lint checks are performed by the [`lint-doc.sh`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/lint-doc.sh) script and can be executed as follows: 1. Navigate to the `gitlab` directory. @@ -168,7 +168,7 @@ You can use markdownlint: [Vale](https://docs.errata.ai/vale/about/) is a grammar, style, and word usage linter for the English language. Vale's configuration is stored in the -[`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini) file located in the root +[`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.vale.ini) file located in the root directory of projects. Vale supports creating [custom tests](https://errata-ai.github.io/vale/styles/) that extend any of @@ -178,7 +178,7 @@ documentation directory of projects. You can find Vale configuration in the following projects: - [`gitlab`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/.vale/gitlab) -- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/-/tree/master/docs/.vale/gitlab) +- [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner/-/tree/main/docs/.vale/gitlab) - [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/tree/master/doc/.vale/gitlab) - [`charts`](https://gitlab.com/gitlab-org/charts/gitlab/-/tree/master/doc/.vale/gitlab) - [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/master/doc/.vale/gitlab) @@ -222,7 +222,7 @@ build pipelines: ``` We recommend installing the version of `markdownlint-cli` - [used](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/.gitlab-ci.yml#L447) when building + [used (see `variables:` section)](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/.gitlab-ci.yml) when building the `image:docs-lint-markdown`. 1. Install [`vale`](https://github.com/errata-ai/vale/releases). For example, to install using @@ -240,7 +240,7 @@ It's important to use linter versions that are the same or newer than those run CI/CD. This provides access to new features and possible bug fixes. To match the versions of `markdownlint-cli` and `vale` used in the GitLab projects, refer to the -[versions used](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/.gitlab-ci.yml#L447) +[versions used (see `variables:` section)](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/.gitlab-ci.yml) when building the `image:docs-lint-markdown` Docker image containing these tools for CI/CD. | Tool | Version | Command | Additional information | @@ -273,7 +273,7 @@ To configure Vale in your editor, install one of the following as appropriate: - Select the **Use CLI** checkbox. - In the **Config** setting, enter an absolute - path to [`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini) + path to [`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.vale.ini) in one of the cloned GitLab repositories on your computer. @@ -330,7 +330,18 @@ document: - To disable all Vale linting rules, add a `` tag before the text, and a `` tag after the text. -Whenever possible, exclude only the problematic rule and line(s). +Whenever possible, exclude only the problematic rule and lines. For more information, see [Vale's documentation](https://docs.errata.ai/vale/scoping#markup-based-configuration). + +### Disable markdownlint tests + +To disable all markdownlint rules, add a `` tag before the text, and a +`` tag after the text. + +To disable only a [specific rule](https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#rules), +add the rule number to the tag, for example `` +and ``. + +Whenever possible, exclude only the problematic lines. diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 8e2028532e4..f035b4d0888 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -85,8 +85,8 @@ If you are a member of the GitLab Slack workspace, you can request help in `#doc ### Reviewing and merging -Anyone with Maintainer access to the relevant GitLab project can merge documentation changes. -Maintainers must make a good-faith effort to ensure that the content: +Anyone with the [Maintainer role](../../user/permissions.md) to the relevant GitLab project can +merge documentation changes. Maintainers must make a good-faith effort to ensure that the content: - Is clear and sufficiently easy for the intended audience to navigate and understand. - Meets the [Documentation Guidelines](index.md) and [Style Guide](styleguide/index.md). @@ -115,7 +115,7 @@ The process involves the following: and link it from the merge request. The process is reflected in the **Documentation** -[merge request template](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/merge_request_templates/Documentation.md). +[merge request template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Documentation.md). ## Other ways to help diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 452b957c705..fb00fe748d0 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -30,7 +30,7 @@ should be added for EE. Licensed features can be stubbed using the spec helper `stub_licensed_features` in `EE::LicenseHelpers`. You can force GitLab to act as CE by either deleting the `ee/` directory or by -setting the [`FOSS_ONLY` environment variable](https://gitlab.com/gitlab-org/gitlab/blob/master/config/helpers/is_ee_env.js) +setting the [`FOSS_ONLY` environment variable](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/helpers/is_ee_env.js) to something that evaluates as `true`. The same works for running tests (for example `FOSS_ONLY=1 yarn jest`). @@ -71,7 +71,7 @@ is applied not only to models. Here's a list of other examples: - `ee/app/views/foo/_bar.html.haml` This works because for every path that is present in CE's eager-load/auto-load -paths, we add the same `ee/`-prepended path in [`config/application.rb`](https://gitlab.com/gitlab-org/gitlab/blob/925d3d4ebc7a2c72964ce97623ae41b8af12538d/config/application.rb#L42-52). +paths, we add the same `ee/`-prepended path in [`config/application.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/925d3d4ebc7a2c72964ce97623ae41b8af12538d/config/application.rb#L42-52). This also applies to views. #### Testing EE-only features @@ -538,7 +538,7 @@ In this case, we could as well just use `render_ce` which would ignore any EE partials. One example would be `ee/app/views/shared/issuable/form/_default_templates.html.haml`: -``` haml +```haml - if @project.feature_available?(:issuable_default_templates) = render_ce 'shared/issuable/form/default_templates' - elsif show_promotions? diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 6b829faf74d..d5f6e95033f 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -36,15 +36,15 @@ Additionally, if you need large repositories or multiple forks for testing, plea ## How does it work? -The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). +The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). -Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! +Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! ## Existing Analyzers/Tokenizers/Filters -These are all defined in [`ee/lib/elastic/latest/config.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/elastic/latest/config.rb) +These are all defined in [`ee/lib/elastic/latest/config.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb) ### Analyzers @@ -214,7 +214,7 @@ end ``` Applied migrations are stored in `gitlab-#{RAILS_ENV}-migrations` index. All migrations not executed -are applied by the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/workers/elastic/migration_worker.rb) +are applied by the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) cron worker sequentially. To update Elastic index mappings, apply the configuration to the respective files: @@ -227,15 +227,15 @@ Any data or index cleanup needed to support migration retries should be handled ### Migration options supported by the `Elastic::MigrationWorker` -[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options: +[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options: -- `batched!` - Allow the migration to run in batches. If set, the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/workers/elastic/migration_worker.rb) +- `batched!` - Allow the migration to run in batches. If set, the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) will re-enqueue itself with a delay which is set using the `throttle_delay` option described below. The batching must be handled within the `migrate` method, this setting controls the re-enqueuing only. - `throttle_delay` - Sets the wait time in between batch runs. This time should be set high enough to allow each migration batch enough time to finish. Additionally, the time should be less than 30 minutes since that is how often the -[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/workers/elastic/migration_worker.rb) +[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) cron worker runs. Default value is 5 minutes. - `pause_indexing!` - Pause indexing while the migration runs. This setting will record the indexing setting before @@ -308,12 +308,15 @@ We choose to use GitLab major version upgrades as a safe time to remove backwards compatibility for indices that have not been fully migrated. We [document this in our upgrade documentation](../update/index.md#upgrading-to-a-new-major-version). We also -choose to remove the migration code and tests so that: +choose to replace the migration code with the halted migration +and remove tests so that: - We don't need to maintain any code that is called from our Advanced Search migrations. - We don't waste CI time running tests for migrations that we don't support anymore. +- Operators who have not run this migration and who upgrade directly to the + target version will see a message prompting them to reindex from scratch. To be extra safe, we will not delete migrations that were created in the last minor version before the major upgrade. So, if we are upgrading to `%14.0`, @@ -334,18 +337,10 @@ For every migration that was created 2 minor versions before the major version being upgraded to, we do the following: 1. Confirm the migration has actually completed successfully for GitLab.com. -1. Replace the content of `migrate` and `completed?` methods as follows: +1. Replace the content of the migration with: ```ruby - def migrate - log_raise "Migration has been deleted in the last major version upgrade." \ - "Migrations are supposed to be finished before upgrading major version https://docs.gitlab.com/ee/update/#upgrading-to-a-new-major-version ." \ - "To correct this issue, recreate your index from scratch: https://docs.gitlab.com/ee/integration/elasticsearch.html#last-resort-to-recreate-an-index." - end - - def completed? - false - end + include Elastic::MigrationObsolete ``` 1. Delete any spec files to support this migration. diff --git a/doc/development/event_tracking/backend.md b/doc/development/event_tracking/backend.md index e8b8e0c4885..3931f0b35ab 100644 --- a/doc/development/event_tracking/backend.md +++ b/doc/development/event_tracking/backend.md @@ -1,8 +1,9 @@ --- redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' +remove_date: '2021-12-01' --- This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). - + diff --git a/doc/development/event_tracking/frontend.md b/doc/development/event_tracking/frontend.md index e8b8e0c4885..3931f0b35ab 100644 --- a/doc/development/event_tracking/frontend.md +++ b/doc/development/event_tracking/frontend.md @@ -1,8 +1,9 @@ --- redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' +remove_date: '2021-12-01' --- This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). - + diff --git a/doc/development/event_tracking/index.md b/doc/development/event_tracking/index.md index e8b8e0c4885..3931f0b35ab 100644 --- a/doc/development/event_tracking/index.md +++ b/doc/development/event_tracking/index.md @@ -1,8 +1,9 @@ --- redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' +remove_date: '2021-12-01' --- This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). - + diff --git a/doc/development/experiment_guide/experimentation.md b/doc/development/experiment_guide/experimentation.md index 7135f8acd9b..ee0f63342f1 100644 --- a/doc/development/experiment_guide/experimentation.md +++ b/doc/development/experiment_guide/experimentation.md @@ -6,10 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Create an A/B test with `Experimentation Module` +NOTE: +We recommend using [GLEX](gitlab_experiment.md) for new experiments. + ## Implement the experiment 1. Add the experiment to the `Gitlab::Experimentation::EXPERIMENTS` hash in - [`experimentation.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib%2Fgitlab%2Fexperimentation.rb): + [`experimentation.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib%2Fgitlab%2Fexperimentation.rb): ```ruby EXPERIMENTS = { diff --git a/doc/development/experiment_guide/gitlab_experiment.md b/doc/development/experiment_guide/gitlab_experiment.md index db9bc521cfd..820d1da0080 100644 --- a/doc/development/experiment_guide/gitlab_experiment.md +++ b/doc/development/experiment_guide/gitlab_experiment.md @@ -394,6 +394,26 @@ You may be asked from time to time to track a specific record ID in experiments. The approach is largely up to the PM and engineer creating the implementation. No recommendations are provided here at this time. +### Record experiment subjects + +Snowplow tracking of identifiable users or groups is prohibited, but you can still +determine if an experiment is successful or not. We're allowed to record the ID of +a namespace, project or user in our database. Therefore, we can tell the experiment +to record their ID together with the assigned experiment variant in the +`experiment_subjects` database table for later analysis. + +For the recording to work, the experiment's context must include a `namespace`, +`group`, `project`, `user`, or `actor`. + +To record the experiment subject when you first assign a variant, call `record!` in +the experiment's block: + +```ruby +experiment(:pill_color, actor: current_user) do |e| + e.record! +end +``` + ## Test with RSpec This gem provides some RSpec helpers and custom matchers. These are in flux as of GitLab 13.10. diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index 0d534a974a1..798c6ff84d0 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -12,7 +12,7 @@ Experiments are run as an A/B/n test, and are behind a feature flag to turn the ## Experiment tracking issue -Each experiment should have an [Experiment tracking](https://gitlab.com/groups/gitlab-org/-/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=growth%20experiment&search=%22Experiment+tracking%22) issue to track the experiment from roll-out through to cleanup/removal. The tracking issue is similar to a feature flag rollout issue, and is also used to track the status of an experiment. Immediately after an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). +Each experiment should have an [Experiment tracking](https://gitlab.com/groups/gitlab-org/-/issues?scope=all&state=opened&label_name[]=growth%20experiment&search=%22Experiment+tracking%22) issue to track the experiment from roll-out through to cleanup/removal. The tracking issue is similar to a feature flag rollout issue, and is also used to track the status of an experiment. Immediately after an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). After the deadline, the issue needs to be resolved and either: - It was successful and the experiment becomes the new default. @@ -46,17 +46,25 @@ One is built into GitLab directly and has been around for a while (this is calle to as `Gitlab::Experiment` -- GLEX for short. Both approaches use [experiment](../feature_flags/index.md#experiment-type) -feature flags, and there is currently no strong suggestion to use one over the other. +feature flags. We recommend using GLEX rather than `Experimentation Module` for new experiments. -| Feature | `Experimentation Module` | GLEX | -| -------------------- |------------------------- | ---- | -| Record user grouping | Yes | No | -| Uses feature flags | Yes | Yes | -| Multivariate (A/B/n) | No | Yes | - -- [Implementing an A/B experiment using `Experimentation Module`](experimentation.md) - [Implementing an A/B/n experiment using GLEX](gitlab_experiment.md) +- [Implementing an A/B experiment using `Experimentation Module`](experimentation.md) Historical Context: `Experimentation Module` was built iteratively with the needs that appeared while implementing Growth sub-department experiments, while GLEX was built with the findings of the team and an easier to use API. + +### Add new icons and illustrations for experiments + +Some experiments may require you to add custom icons or illustrations to our codebase. +This process is lengthy and at this stage, the outcome of the experiment uncertain. +Therefore, you should postpone this effort until the [experiment cleanup process](https://about.gitlab.com/handbook/engineering/development/growth/#experiment-cleanup-issue). + +We recommend the following workflow: + +1. Add an icon or illustration as an `.svg` file in the `/app/assets/images` (or EE) path in the GitLab repository. +1. Use `image_tag` or `image_path` to render it via the asset pipeline. +1. **If the experiment is a success**, designers add the new icon or illustration to the Pajamas UI kit as part of the cleanup process. + Engineers can then add it to the [SVG library](https://gitlab-org.gitlab.io/gitlab-svgs/) and modify the implementation based on the + [Frontend Development Guidelines](../fe_guide/icons.md#usage-in-hamlrails-2). diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index ab1325c67a9..15818941b24 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -39,9 +39,20 @@ so when in doubt don't use `aria-*`, `role`, and `tabindex` and stick with seman - [Clickable icons](#icons-that-are-clickable) are buttons, that is, `` is used and not ``. - Icon-only buttons have an `aria-label`. - Interactive elements can be [accessed with the Tab key](#support-keyboard-only-use) and have a visible focus state. +- Elements with [tooltips](#tooltips) are focusable using the Tab key. - Are any `role`, `tabindex` or `aria-*` attributes unnecessary? - Can any `div` or `span` elements be replaced with a more semantic [HTML element](https://developer.mozilla.org/en-US/docs/Web/HTML/Element) like `p`, `button`, or `time`? +## Provide a good document outline + +[Headings are the primary mechanism used by screen reader users to navigate content](https://webaim.org/projects/screenreadersurvey8/#finding). +Therefore, the structure of headings on a page should make sense, like a good table of contents. +We should ensure that: + +- There is only one `h1` element on the page. +- Heading levels are not skipped. +- Heading levels are nested correctly. + ## Provide accessible names for screen readers To provide markup with accessible names, ensure every: @@ -257,6 +268,9 @@ Image examples: + + + ``` #### Buttons and links with descriptive accessible names @@ -275,6 +289,14 @@ Buttons and links should have accessible names that are descriptive enough to be {{ __("GitLab's accessibility page") }} ``` +#### Links styled like buttons + +Links can be styled like buttons using `GlButton`. + +```html + {{ __('Link styled as a button') }} +``` + ## Role In general, avoid using `role`. @@ -336,7 +358,7 @@ Once the markup is semantically complete, use CSS to update it to its desired vi
Expand
-Expand +Expand ``` ### Do not use `tabindex="0"` on interactive elements @@ -423,6 +445,30 @@ Icons that are clickable are semantically buttons, so they should be rendered as ``` +## Tooltips + +When adding tooltips, we must ensure that the element with the tooltip can receive focus so keyboard users can see the tooltip. +If the element is a static one, such as an icon, we can enclose it in a button, which already is +focusable, so we don't have to add `tabindex=0` to the icon. + +The following code snippet is a good example of an icon with a tooltip. + +- It is automatically focusable, as it is a button. +- It is given an accessible name with `aria-label`, as it is a button with no text. +- We can use the `gl-hover-bg-transparent!` class if we don't want the button's background to become gray on hover. +- We can use the `gl-p-0!` class to remove the button padding, if needed. + +```html + +``` + ## Hiding elements Use the following table to hide elements from users, when appropriate. @@ -478,5 +524,3 @@ We have two options for Web accessibility testing: - [The A11Y Project](https://www.a11yproject.com/) is a good resource for accessibility - [Awesome Accessibility](https://github.com/brunopulis/awesome-a11y) is a compilation of accessibility-related material -- You can read [Chrome Accessibility Developer Tools'](https://github.com/GoogleChrome/accessibility-developer-tools) - rules on its [Audit Rules page](https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules) diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md new file mode 100644 index 00000000000..f6329f39636 --- /dev/null +++ b/doc/development/fe_guide/content_editor.md @@ -0,0 +1,116 @@ +--- +stage: Create +group: Editor +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Content Editor **(FREE)** + +The Content Editor is a UI component that provides a WYSIWYG editing +experience for [GitLab Flavored Markdown](../../user/markdown.md) (GFM) in the GitLab application. +It also serves as the foundation for implementing Markdown-focused editors +that target other engines, like static site generators. + +We use [tiptap 2.0](https://www.tiptap.dev/) and [ProseMirror](https://prosemirror.net/) +to build the Content Editor. These frameworks provide a level of abstraction on top of +the native +[`contenteditable`](https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Editable_content) web technology. + +## Architecture remarks + +At a high level, the Content Editor: + +- Imports arbitrary Markdown. +- Renders it in a HTML editing area. +- Exports it back to Markdown with changes introduced by the user. + +The Content Editor relies on the +[Markdown API endpoint](../../api/markdown.md) to transform Markdown +into HTML. It sends the Markdown input to the REST API and displays the API's +HTML output in the editing area. The editor exports the content back to Markdown +using a client-side library that serializes editable documents into Markdown. + +![Content Editor high level diagram](img/content_editor_highlevel_diagram.png) + +Check the [Content Editor technical design document](https://docs.google.com/document/d/1fKOiWpdHned4KOLVOOFYVvX1euEjMP5rTntUhpapdBg) +for more information about the design decisions that drive the development of the editor. + +**NOTE**: We also designed the Content Editor to be extensible. We intend to provide +more information about extension development for supporting new types of content in upcoming +milestones. + +## GitLab Flavored Markdown support + +The [GitLab Flavored Markdown](../../user/markdown.md) extends +the [CommonMark specification](https://spec.commonmark.org/0.29/) with support for a +variety of content types like diagrams, math expressions, and tables. Supporting +all GitLab Flavored Markdown content types in the Content Editor is a work in progress. For +the status of the ongoing development for CommonMark and GitLab Flavored Markdown support, read: + +- [Basic Markdown formatting extensions](https://gitlab.com/groups/gitlab-org/-/epics/5404) epic. +- [GitLab Flavored Markdown extensions](https://gitlab.com/groups/gitlab-org/-/epics/5438) epic. + +## Usage + +To include the Content Editor in your feature, import the `createContentEditor` factory +function and the `ContentEditor` Vue component. `createContentEditor` sets up an instance +of [tiptap's Editor class](https://www.tiptap.dev/api/editor) with all the necessary +extensions to support editing GitLab Flavored Markdown content. It also creates +a Markdown serializer that allows exporting tiptap's document format to Markdown. + +`createContentEditor` requires a `renderMarkdown` parameter invoked +by the editor every time it needs to convert Markdown to HTML. The Content Editor +does not provide a default value for this function yet. + +**NOTE**: The Content Editor is in an early development stage. Usage and development +guidelines are subject to breaking changes in the upcoming months. + +```html + + +``` + +Call `setSerializedContent` to set initial Markdown in the Editor. This method is +asynchronous because it makes an API request to render the Markdown input. +`getSerializedContent` returns a Markdown string that represents the serialized +version of the editable document. diff --git a/doc/development/fe_guide/design_anti_patterns.md b/doc/development/fe_guide/design_anti_patterns.md index ee4fceff927..0788921fce4 100644 --- a/doc/development/fe_guide/design_anti_patterns.md +++ b/doc/development/fe_guide/design_anti_patterns.md @@ -119,8 +119,8 @@ Here are some ills that Singletons often produce: such as no clear ownership and no access control. These leads to high coupling situations that can be buggy and difficult to untangle. 1. **Infectious.** Singletons are infectious, especially when they manage state. Consider the component - [RepoEditor](https://gitlab.com/gitlab-org/gitlab/blob/27ad6cb7b76430fbcbaf850df68c338d6719ed2b/app%2Fassets%2Fjavascripts%2Fide%2Fcomponents%2Frepo_editor.vue#L0-1) - used in the Web IDE. This component interfaces with a Singleton [Editor](https://gitlab.com/gitlab-org/gitlab/blob/862ad57c44ec758ef3942ac2e7a2bd40a37a9c59/app%2Fassets%2Fjavascripts%2Fide%2Flib%2Feditor.js#L21) + [RepoEditor](https://gitlab.com/gitlab-org/gitlab/-/blob/27ad6cb7b76430fbcbaf850df68c338d6719ed2b/app%2Fassets%2Fjavascripts%2Fide%2Fcomponents%2Frepo_editor.vue#L0-1) + used in the Web IDE. This component interfaces with a Singleton [Editor](https://gitlab.com/gitlab-org/gitlab/-/blob/862ad57c44ec758ef3942ac2e7a2bd40a37a9c59/app%2Fassets%2Fjavascripts%2Fide%2Flib%2Feditor.js#L21) which manages some state for working with Monaco. Because of the Singleton nature of the Editor class, the component `RepoEditor` is now forced to be a Singleton as well. Multiple instances of this component would cause production issues because no one truly owns the instance of `Editor`. diff --git a/doc/development/fe_guide/editor_lite.md b/doc/development/fe_guide/editor_lite.md index 5ad0c753ced..f28588c23e9 100644 --- a/doc/development/fe_guide/editor_lite.md +++ b/doc/development/fe_guide/editor_lite.md @@ -15,6 +15,7 @@ GitLab features use it, including: - [CI Linter](../../ci/lint.md) - [Snippets](../../user/snippets.md) - [Web Editor](../../user/project/repository/web_editor.md) +- [Security Policies](../../user/application_security/threat_monitoring/index.md) ## How to use Editor Lite diff --git a/doc/development/fe_guide/event_tracking.md b/doc/development/fe_guide/event_tracking.md index e8b8e0c4885..3931f0b35ab 100644 --- a/doc/development/fe_guide/event_tracking.md +++ b/doc/development/fe_guide/event_tracking.md @@ -1,8 +1,9 @@ --- redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' +remove_date: '2021-12-01' --- This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). - + diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index bf1dae6e7bd..6b9d5ace4e6 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -32,15 +32,15 @@ question: document.body.dataset.page ``` -Find here the [source code setting the attribute](https://gitlab.com/gitlab-org/gitlab/blob/cc5095edfce2b4d4083a4fb1cdc7c0a1898b9921/app/views/layouts/application.html.haml#L4). +Find here the [source code setting the attribute](https://gitlab.com/gitlab-org/gitlab/-/blob/cc5095edfce2b4d4083a4fb1cdc7c0a1898b9921/app/views/layouts/application.html.haml#L4). #### Rails routes -The `rake routes` command can be used to list all the routes available in the application. Piping the output into `grep`, we can perform a search through the list of available routes. +The `rails routes` command can be used to list all the routes available in the application. Piping the output into `grep`, we can perform a search through the list of available routes. The output includes the request types available, route parameters and the relevant controller. ```shell -bundle exec rake routes | grep "issues" +bundle exec rails routes | grep "issues" ``` ### 2. `modal_copy_button` vs `clipboard_button` @@ -82,7 +82,7 @@ follow up issue and attach it to the component implementation epic found in the ### 4. My submit form button becomes disabled after submitting -A Submit button inside of a form attaches an `onSubmit` event listener on the form element. [This code](https://gitlab.com/gitlab-org/gitlab/blob/794c247a910e2759ce9b401356432a38a4535d49/app/assets/javascripts/main.js#L225) adds a `disabled` class selector to the submit button when the form is submitted. To avoid this behavior, add the class `js-no-auto-disable` to the button. +A Submit button inside of a form attaches an `onSubmit` event listener on the form element. [This code](https://gitlab.com/gitlab-org/gitlab/-/blob/794c247a910e2759ce9b401356432a38a4535d49/app/assets/javascripts/main.js#L225) adds a `disabled` class selector to the submit button when the form is submitted. To avoid this behavior, add the class `js-no-auto-disable` to the button. ### 5. Should one use a full URL (for example `gon.gitlab_url`) or a full path (for example `gon.relative_url_root`) when referencing backend endpoints? @@ -172,7 +172,7 @@ To return to the normal development mode: ### 8. Babel polyfills -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/28837) in GitLab 12.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28837) in GitLab 12.8. GitLab has enabled the Babel `preset-env` option [`useBuiltIns: 'usage'`](https://babeljs.io/docs/en/babel-preset-env#usebuiltins-usage). diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 49c511c2b85..870605c82f4 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -94,7 +94,7 @@ their execution by clicking **Execute query** button on the top left: ## Apollo Client To save duplicated clients getting created in different apps, we have a -[default client](https://gitlab.com/gitlab-org/gitlab/blob/master/app/assets/javascripts/lib/graphql.js) that should be used. This sets up the +[default client](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/lib/graphql.js) that should be used. This sets up the Apollo client with the correct URL and also sets the CSRF headers. Default client accepts two parameters: `resolvers` and `config`. @@ -106,6 +106,12 @@ Default client accepts two parameters: `resolvers` and `config`. - `assumeImmutableResults` (set to `false` by default) - this setting, when set to `true`, assumes that every single operation on updating Apollo Cache is immutable. It also sets `freezeResults` to `true`, so any attempt on mutating Apollo Cache throws a console warning in development environment. Please ensure you're following the immutability pattern on cache update operations before setting this option to `true`. - `fetchPolicy` determines how you want your component to interact with the Apollo cache. Defaults to "cache-first". +### Multiple client queries for the same object + +If you are make multiple queries to the same Apollo client object you might encounter the following error: "Store error: the application attempted to write an object with no provided ID but the store already contains an ID of SomeEntity". [This error only should occur when you have made a query with an ID field for a portion, then made another that returns what would be the same object, but is missing the ID field.](https://github.com/apollographql/apollo-client/issues/2510#issue-271829009) + +Please note this is being tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/326101) and the documentation will be updated when this issue is resolved. + ## GraphQL Queries To save query compilation at runtime, webpack can directly import `.graphql` @@ -1091,7 +1097,7 @@ it('renders a loading state', () => { const mockApollo = createMockApolloProvider(); const wrapper = createComponent({ mockApollo }); - expect(wrapper.find(LoadingSpinner).exists()).toBe(true) + expect(wrapper.findComponent(LoadingSpinner).exists()).toBe(true) }); it('renders designs list', async () => { @@ -1393,7 +1399,6 @@ describe('My Index test with `createMockApollo`', () => { afterEach(() => { wrapper.destroy(); - wrapper = null; fetchLocalUserSpy = null; }); diff --git a/doc/development/fe_guide/img/content_editor_highlevel_diagram.png b/doc/development/fe_guide/img/content_editor_highlevel_diagram.png new file mode 100644 index 00000000000..73a71cf5843 Binary files /dev/null and b/doc/development/fe_guide/img/content_editor_highlevel_diagram.png differ diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 0f3754c29e7..00f0d72571a 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -93,6 +93,11 @@ General information about frontend [dependencies](dependencies.md) and how we ma How we implement [keyboard shortcuts](keyboard_shortcuts.md) that can be customized and disabled. +## Editors + +GitLab text editing experiences are provided by the [Source Editor](editor_lite.md) and +the [Content Editor](content_editor.md). + ## Frontend FAQ Read the [frontend's FAQ](frontend_faq.md) for common small pieces of helpful information. diff --git a/doc/development/fe_guide/style/vue.md b/doc/development/fe_guide/style/vue.md index 93e4f234ccb..5c79d47e7b0 100644 --- a/doc/development/fe_guide/style/vue.md +++ b/doc/development/fe_guide/style/vue.md @@ -463,7 +463,7 @@ Creating a global, mutable wrapper provides a number of advantages, including th let wrapper; // this can now be reused across tests - const findMyComponent = wrapper.find(MyComponent); + const findMyComponent = wrapper.findComponent(MyComponent); // ... }) ``` @@ -565,16 +565,15 @@ the mounting function (`mount` or `shallowMount`) to be used to mount the compon function createComponent({ mountFn = shallowMount } = {}) { } ``` -1. Wrap calls to `mount` and `shallowMount` in `extendedWrapper`, this exposes `wrapper.findByTestId()`: +1. Use the `mountExtended` and `shallowMountExtended` helpers to expose `wrapper.findByTestId()`: ```javascript - import { shallowMount } from '@vue/test-utils'; - import { extendedWrapper } from 'helpers/vue_test_utils_helper'; + import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import { SomeComponent } from 'components/some_component.vue'; let wrapper; - const createWrapper = () => { wrapper = extendedWrapper(shallowMount(SomeComponent)); }; + const createWrapper = () => { wrapper = shallowMountExtended(SomeComponent); }; const someButton = () => wrapper.findByTestId('someButtonTestId'); ``` diff --git a/doc/development/fe_guide/troubleshooting.md b/doc/development/fe_guide/troubleshooting.md index 1b3991ee80d..028184e0397 100644 --- a/doc/development/fe_guide/troubleshooting.md +++ b/doc/development/fe_guide/troubleshooting.md @@ -27,15 +27,15 @@ See [this video](https://youtu.be/-BkEhghP-kM) for an in-depth overview and inve **Remedy - Try cloning the object that has Vue watchers** ```patch -- expect(wrapper.find(ChildComponent).props()).toEqual(...); -+ expect(cloneDeep(wrapper.find(ChildComponent).props())).toEqual(...) +- expect(wrapper.findComponent(ChildComponent).props()).toEqual(...); ++ expect(cloneDeep(wrapper.findComponent(ChildComponent).props())).toEqual(...) ``` **Remedy - Try using `toMatchObject` instead of `toEqual`** ```patch -- expect(wrapper.find(ChildComponent).props()).toEqual(...); -+ expect(wrapper.find(ChildComponent).props()).toMatchObject(...); +- expect(wrapper.findComponent(ChildComponent).props()).toEqual(...); ++ expect(wrapper.findComponent(ChildComponent).props()).toMatchObject(...); ``` Please note that `toMatchObject` actually changes the nature of the assertion and won't fail if some items are **missing** from the expectation. diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 1cce699218c..0a769f257d0 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -13,7 +13,7 @@ To get started with Vue, read through [their documentation](https://vuejs.org/v2 What is described in the following sections can be found in these examples: - [Web IDE](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/ide/stores) -- [Security products](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports) +- [Security products](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/assets/javascripts/vue_shared/security_reports) - [Registry](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/assets/javascripts/registry/stores) ## Vue architecture @@ -323,17 +323,13 @@ testing the rendered output. Here's an example of a well structured unit test for [this Vue component](#appendix---vue-component-subject-under-test): ```javascript -import { shallowMount } from '@vue/test-utils'; -import { extendedWrapper } from 'helpers/vue_test_utils_helper'; import { GlLoadingIcon } from '@gitlab/ui'; import MockAdapter from 'axios-mock-adapter'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import axios from '~/lib/utils/axios_utils'; import App from '~/todos/app.vue'; -const TEST_TODOS = [ - { text: 'Lorem ipsum test text' }, - { text: 'Lorem ipsum 2' }, -]; +const TEST_TODOS = [{ text: 'Lorem ipsum test text' }, { text: 'Lorem ipsum 2' }]; const TEST_NEW_TODO = 'New todo title'; const TEST_TODO_PATH = '/todos'; @@ -351,28 +347,27 @@ describe('~/todos/app.vue', () => { afterEach(() => { // IMPORTANT: Clean up the component instance and axios mock adapter wrapper.destroy(); - wrapper = null; - mock.restore(); }); // It is very helpful to separate setting up the component from // its collaborators (for example, Vuex and axios). const createWrapper = (props = {}) => { - wrapper = extendedWrapper( - shallowMount(App, { - propsData: { - path: TEST_TODO_PATH, - ...props, - }, - }) - ); + wrapper = shallowMountExtended(App, { + propsData: { + path: TEST_TODO_PATH, + ...props, + }, + }); }; // Helper methods greatly help test maintainability and readability. - const findLoader = () => wrapper.find(GlLoadingIcon); + const findLoader = () => wrapper.findComponent(GlLoadingIcon); const findAddButton = () => wrapper.findByTestId('add-button'); const findTextInput = () => wrapper.findByTestId('text-input'); - const findTodoData = () => wrapper.findAll('[data-testid="todo-item"]').wrappers.map(wrapper => ({ text: wrapper.text() })); + const findTodoData = () => + wrapper + .findAllByTestId('todo-item') + .wrappers.map((item) => ({ text: item.text() })); describe('when mounted and loading', () => { beforeEach(() => { @@ -401,14 +396,13 @@ describe('~/todos/app.vue', () => { expect(findTodoData()).toEqual(TEST_TODOS); }); - it('when todo is added, should post new todo', () => { - findTextInput().vm.$emit('update', TEST_NEW_TODO) + it('when todo is added, should post new todo', async () => { + findTextInput().vm.$emit('update', TEST_NEW_TODO); findAddButton().vm.$emit('click'); - return wrapper.vm.$nextTick() - .then(() => { - expect(mock.history.post.map(x => JSON.parse(x.data))).toEqual([{ text: TEST_NEW_TODO }]); - }); + await wrapper.vm.$nextTick(); + + expect(mock.history.post.map((x) => JSON.parse(x.data))).toEqual([{ text: TEST_NEW_TODO }]); }); }); }); diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index d44ab64ae5d..3d0044928f1 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -40,7 +40,7 @@ When using Vuex at GitLab, separate these concerns into different files to impro The following example shows an application that lists and adds users to the state. (For a more complex example implementation, review the security -applications stored in this [repository](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/app/assets/javascripts/vue_shared/security_reports/store)). +applications stored in this [repository](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/assets/javascripts/vue_shared/security_reports/store)). ### `index.js` @@ -464,7 +464,6 @@ describe('component', () => { afterEach(() => { wrapper.destroy(); - wrapper = null; }); it('should show a user', async () => { diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 08a4401181b..a9ebcfc9fba 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -99,7 +99,7 @@ Guidelines: Before toggling any feature flag, check that there are no ongoing significant incidents on GitLab.com. You can do this by checking the `#production` and `#incident-management` Slack channels, or looking for -[open incident issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=incident) +[open incident issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?scope=all&state=opened&label_name[]=incident) (although check the dates and times). We do not want to introduce changes during an incident, as it can make @@ -213,9 +213,6 @@ actors. Feature.enabled?(:some_feature, group) ``` -**Percentage of time** rollout is not a good idea if what you want is to make sure a feature -is always on or off to the users. In that case, **Percentage of actors** rollout is a better method. - Lastly, to verify that the feature is deemed stable in as many cases as possible, you should fully roll out the feature by enabling the flag **globally** by running: @@ -226,6 +223,32 @@ you should fully roll out the feature by enabling the flag **globally** by runni This changes the feature flag state to be **enabled** always, which overrides the existing gates (e.g. `--group=gitlab-org`) in the above processes. +Note, that if an actor based feature gate is present, switching the +`default_enabled` attribute of the YAML definition from `false` to `true` +will not have any effect. The feature gate must be deleted first. + +For example, a feature flag is set via chatops: + +```shell +/chatops run feature set --project=gitlab-org/gitlab some_feature true +``` + +When the `default_enabled` attribute in the YAML definition is switched to +`true`, the feature gate must be deleted to have the desired effect: + +```shell +/chatops run feature delete some_feature +``` + +##### Percentage of actors vs percentage of time rollouts + +If you want to make sure a feature is always on or off for users, use a **Percentage of actors** +rollout. Avoid using percentage of _time_ rollouts in this case. + +A percentage of _time_ rollout can introduce inconsistent behavior when `Feature.enabled?` +is used multiple times in the code because the feature flag value is randomized each time +`Feature.enabled?` is called on your code path. + ##### Disabling feature flags To disable a feature flag that has been globally enabled you can run: @@ -250,7 +273,7 @@ Any feature flag change that affects GitLab.com (production) via [ChatOps](https is automatically logged in an issue. The issue is created in the -[gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/-/issues?scope=all&utf8=%E2%9C%93&state=closed) +[gl-infra/feature-flag-log](https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/-/issues?scope=all&state=closed) project, and it will at minimum log the Slack handle of person enabling a feature flag, the time, and the name of the flag being changed. diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index 79efd6d5502..d7807c6f586 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -1,7 +1,9 @@ --- redirect_to: 'index.md' +remove_date: '2021-06-01' --- This document was moved to [another location](index.md). + \ No newline at end of file diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index e18bcaa1f4e..79a100e44a5 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -41,7 +41,7 @@ should be leveraged: 1. [Create a new feature flag](#create-a-new-feature-flag) which is **off** by default, in the first merge request which uses the flag. - Flags [should not be added separately](#risk-of-a-broken-master-main-branch). + Flags [should not be added separately](#risk-of-a-broken-main-branch). 1. Submit incremental changes via one or more merge requests, ensuring that any new code added can only be reached if the feature flag is **on**. You can keep the feature flag enabled on your local GDK during development. @@ -59,11 +59,11 @@ flag does not have to stick around for a specific amount of time is deemed stable. Stable means it works on GitLab.com without causing any problems, such as outages. -## Risk of a broken master (main) branch +## Risk of a broken main branch -Feature flags **must** be used in the MR that introduces them. Not doing so causes a -[broken master](https://about.gitlab.com/handbook/engineering/workflow/#broken-master) scenario due -to the `rspec:feature-flags` job that only runs on the `master` branch. +Feature flags must be used in the MR that introduces them. Not doing so causes a +[broken main branch](https://about.gitlab.com/handbook/engineering/workflow/#broken-master) scenario due +to the `rspec:feature-flags` job that only runs on the `main` branch. ## Types of feature flags @@ -635,7 +635,7 @@ with how it interacts with `ActiveRecord`. ### End-to-end (QA) tests Toggling feature flags works differently in end-to-end (QA) tests. The end-to-end test framework does not have direct access to -Rails or the database, so it can't use Flipper. Instead, it uses [the public API](../../api/features.md#set-or-create-a-feature). Each end-to-end test can [enable or disable a feature flag during the test](../testing_guide/end_to_end/feature_flags.md). Alternatively, you can enable or disable a feature flag before one or more tests when you [run them from your GitLab repository's `qa` directory](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled-or-disabled), or if you [run the tests via GitLab QA](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#running-tests-with-a-feature-flag-enabled). +Rails or the database, so it can't use Flipper. Instead, it uses [the public API](../../api/features.md#set-or-create-a-feature). Each end-to-end test can [enable or disable a feature flag during the test](../testing_guide/end_to_end/feature_flags.md). Alternatively, you can enable or disable a feature flag before one or more tests when you [run them from your GitLab repository's `qa` directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa#running-tests-with-a-feature-flag-enabled-or-disabled), or if you [run the tests via GitLab QA](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#running-tests-with-a-feature-flag-enabled). [As noted above, feature flags are not enabled by default in end-to-end tests.](#feature-flags-in-tests) This means that end-to-end tests will run with feature flags in the default state implemented in the source diff --git a/doc/development/feature_flags/process.md b/doc/development/feature_flags/process.md index 247dafe9f0b..0e962218ab9 100644 --- a/doc/development/feature_flags/process.md +++ b/doc/development/feature_flags/process.md @@ -1,5 +1,6 @@ --- redirect_to: 'https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/' +remove_date: '2021-06-01' --- This document was moved to [another location](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/). diff --git a/doc/development/file_storage.md b/doc/development/file_storage.md index 1f929d64058..71fc81a6ea3 100644 --- a/doc/development/file_storage.md +++ b/doc/development/file_storage.md @@ -60,7 +60,7 @@ hash of the project ID instead, if project migrates to the new approach (introdu We provide an [all-in-one Rake task](../administration/raketasks/uploads/migrate.md) to migrate all uploads to object storage in one go. If a new Uploader class or model type is introduced, make sure you add a Rake task invocation corresponding to it to the -[category list](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/uploads/migrate.rake). +[category list](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/tasks/gitlab/uploads/migrate.rake). ### Path segments diff --git a/doc/development/geo.md b/doc/development/geo.md index 05fadcad08a..8017bd21126 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -199,8 +199,8 @@ needs to be applied to the tracking database on each **secondary** node. ### Configuration -The database configuration is set in [`config/database_geo.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/config/database_geo.yml.postgresql). -The directory [`ee/db/geo`](https://gitlab.com/gitlab-org/gitlab/tree/master/ee/db/geo) +The database configuration is set in [`config/database_geo.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database_geo.yml.postgresql). +The directory [`ee/db/geo`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/geo) contains the schema and migrations for this database. To write a migration for the database, use the `GeoMigrationGenerator`: @@ -217,7 +217,7 @@ bundle exec rake geo:db:migrate ## Finders -Geo uses [Finders](https://gitlab.com/gitlab-org/gitlab/tree/master/app/finders), +Geo uses [Finders](https://gitlab.com/gitlab-org/gitlab/-/tree/master/app/finders), which are classes take care of the heavy lifting of looking up projects/attachments/etc. in the tracking database and main database. @@ -320,7 +320,7 @@ The process running on the **secondary** node that looks for new ### `Gitlab::Geo` utilities Small utility methods related to Geo go into the -[`ee/lib/gitlab/geo.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/geo.rb) +[`ee/lib/gitlab/geo.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/geo.rb) file. Many of these methods are cached using the `RequestStore` class, to diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 1513b65d19f..ad24353fde8 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -159,7 +159,7 @@ In some cases, such as building a Go project for it to act as a dependency of a CI run for another project, removing the `vendor/` directory means the code must be downloaded repeatedly, which can lead to intermittent problems due to rate limiting or network failures. In these circumstances, you should [cache the -downloaded code between](../../ci/caching/index.md#caching-go-dependencies). +downloaded code between](../../ci/caching/index.md#cache-go-dependencies). There was a [bug on modules checksums](https://github.com/golang/go/issues/29278) in Go versions earlier than v1.11.4, so make diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index a506b67d89d..40598eaff95 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -196,7 +196,7 @@ RuboCop](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-4-stable/.rubocop.yml# Using the inline `:javascript` Haml filters comes with a performance overhead. Using inline JavaScript is not a good way to structure your code and should be avoided. -We've [removed these two filters](https://gitlab.com/gitlab-org/gitlab/blob/master/config/initializers/hamlit.rb) +We've [removed these two filters](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/initializers/hamlit.rb) in an initializer. ### Further reading diff --git a/doc/development/graphql_guide/authorization.md b/doc/development/graphql_guide/authorization.md index 8f17a8b6c93..a317b5d805b 100644 --- a/doc/development/graphql_guide/authorization.md +++ b/doc/development/graphql_guide/authorization.md @@ -40,7 +40,7 @@ to filter the records. This minimizes database queries and unnecessary authorization checks of the loaded records. It also avoids situations, such as short pages, which can expose the presence of confidential resources. -See [`authorization_spec.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/graphql/features/authorization_spec.rb) +See [`authorization_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/features/authorization_spec.rb) for examples of all the authorization schemes discussed here. ## Type authorization diff --git a/doc/development/graphql_guide/pagination.md b/doc/development/graphql_guide/pagination.md index 5db9238faed..5fd2179ea9b 100644 --- a/doc/development/graphql_guide/pagination.md +++ b/doc/development/graphql_guide/pagination.md @@ -223,6 +223,26 @@ the `order_due_date_and_labels_priority` method creates a very complex query. These types of queries are not supported. In these instances, you can use offset pagination. +#### Gotchas + +Do not define a collection's order using the string syntax: + +```ruby +# Bad +items.order('created_at DESC') +``` + +Instead, use the hash syntax: + +```ruby +# Good +items.order(created_at: :desc) +``` + +The first example won't correctly embed the sort information (`created_at`, in +the example above) into the pagination cursors, which will result in an +incorrect sort order. + ### Offset pagination There are times when the [complexity of sorting](#limitations-of-query-complexity) @@ -267,7 +287,7 @@ For consistency, we manually set the pagination cursors based on values returned You can see an example implementation in the following files: - [`types/error__tracking/sentry_error_collection_type.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/types/error_tracking/sentry_error_collection_type.rb) which adds an extension to `field :errors`. -- [`resolvers/error_tracking/sentry_errors_resolver.rb`](https://gitlab.com/gitlab-org/gitlab/blob/master/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb) which returns the data from the resolver. +- [`resolvers/error_tracking/sentry_errors_resolver.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb) which returns the data from the resolver. ## Testing diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index b177a7e0138..7ea8378b6db 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -10,16 +10,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w For working with internationalization (i18n), [GNU gettext](https://www.gnu.org/software/gettext/) is used given it's the most -used tool for this task and there are a lot of applications that help us -work with it. +used tool for this task and there are many applications that help us work with it. NOTE: -All `rake` commands described on this page must be run on a GitLab instance, usually GDK. +All `rake` commands described on this page must be run on a GitLab instance. This instance is +usually the GitLab Development Kit (GDK). -## Setting up GitLab Development Kit (GDK) +## Setting up the GitLab Development Kit (GDK) -In order to be able to work on the [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss) -project you must download and configure it through [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/set-up-gdk.md). +To work on the [GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-foss) +project, you must download and configure it through the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/set-up-gdk.md). After you have the GitLab project ready, you can start working on the translation. @@ -27,34 +27,33 @@ After you have the GitLab project ready, you can start working on the translatio The following tools are used: -1. [`gettext_i18n_rails`](https://github.com/grosser/gettext_i18n_rails): this - gem allow us to translate content from models, views and controllers. Also - it gives us access to the following Rake tasks: - - `rake gettext:find`: Parses almost all the files from the - Rails application looking for content that has been marked for - translation. Finally, it updates the PO files with the new content that - it has found. - - `rake gettext:pack`: Processes the PO files and generates the - MO files that are binary and are finally used by the application. - -1. [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): - this gem is useful to make the translations available in JavaScript. It - provides the following Rake task: - - `rake gettext:po_to_json`: Reads the contents from the PO files and - generates JSON files containing all the available translations. - -1. PO editor: there are multiple applications that can help us to work with PO - files, a good option is [Poedit](https://poedit.net/download) which is - available for macOS, GNU/Linux and Windows. +- [`gettext_i18n_rails`](https://github.com/grosser/gettext_i18n_rails): + this gem allows us to translate content from models, views, and controllers. It also gives us + access to the following Rake tasks: + + - `rake gettext:find`: parses almost all the files from the Rails application looking for content + marked for translation. It then updates the PO files with this content. + - `rake gettext:pack`: processes the PO files and generates the binary MO files that the + application uses. + +- [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): + this gem makes the translations available in JavaScript. It provides the following Rake task: + + - `rake gettext:po_to_json`: reads the contents of the PO files and generates JSON files that + contain all the available translations. + +- PO editor: there are multiple applications that can help us work with PO files. A good option is + [Poedit](https://poedit.net/download), + which is available for macOS, GNU/Linux, and Windows. ## Preparing a page for translation -We basically have 4 types of files: +There are four file types: -1. Ruby files: basically Models and Controllers. -1. HAML files: these are the view files. -1. ERB files: used for email templates. -1. JavaScript files: we mostly need to work with Vue templates. +- Ruby files: models and controllers. +- HAML files: view files. +- ERB files: used for email templates. +- JavaScript files: we mostly work with Vue templates. ### Ruby files @@ -72,7 +71,7 @@ Or: hello = "Hello world!" ``` -You can easily mark that content for translation with: +You can mark that content for translation with: ```ruby def hello @@ -86,26 +85,21 @@ Or: hello = _("Hello world!") ``` -Be careful when translating strings at the class or module level since these would only be -evaluated once at class load time. - -For example: +Be careful when translating strings at the class or module level since these are only evaluated once +at class load time. For example: ```ruby validates :group_id, uniqueness: { scope: [:project_id], message: _("already shared with this group") } ``` -This would be translated when the class is loaded and result in the error message -always being in the default locale. - -Active Record's `:message` option accepts a `Proc`, so we can do this instead: +This is translated when the class loads and results in the error message always being in the default +locale. Active Record's `:message` option accepts a `Proc`, so do this instead: ```ruby validates :group_id, uniqueness: { scope: [:project_id], message: -> (object, data) { _("already shared with this group") } } ``` -Messages in the API (`lib/api/` or `app/graphql`) do -not need to be externalized. +Messages in the API (`lib/api/` or `app/graphql`) do not need to be externalized. ### HAML files @@ -145,13 +139,20 @@ import { __ } from '~/locale'; const label = __('Subscribe'); ``` -In order to test JavaScript translations you have to change the GitLab -localization to another language than English and you have to generate JSON files -using `bin/rake gettext:po_to_json` or `bin/rake gettext:compile`. +To test JavaScript translations you must: + +- Change the GitLab localization to a language other than English. +- Generate JSON files by using `bin/rake gettext:po_to_json` or `bin/rake gettext:compile`. ### Vue files -In Vue files we make both the `__()` (double underscore parenthesis) function and the `s__()` (namespaced double underscore parenthesis) function available that you can import from the `~/locale` file. For instance: +In Vue files, we make the following functions available: + +- `__()` (double underscore parenthesis) +- `s__()` (namespaced double underscore parenthesis) + +You can therefore import from the `~/locale` file. +For example: ```javascript import { __, s__ } from '~/locale'; @@ -228,24 +229,24 @@ For the static text strings we suggest two patterns for using these translations ``` -In order to visually test the Vue translations you have to change the GitLab -localization to another language than English and you have to generate JSON files -using `bin/rake gettext:po_to_json` or `bin/rake gettext:compile`. +To visually test the Vue translations: -### Dynamic translations +1. Change the GitLab localization to another language than English. +1. Generate JSON files using `bin/rake gettext:po_to_json` or `bin/rake gettext:compile`. -Sometimes there are some dynamic translations that can't be found by the -parser when running `bin/rake gettext:find`. For these scenarios you can -use the [`N_` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). +### Dynamic translations -There is also and alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). +Sometimes there are dynamic translations that the parser can't find when running +`bin/rake gettext:find`. For these scenarios you can use the [`N_` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). +There's also an alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). ## Working with special content ### Interpolation -Placeholders in translated text should match the code style of the respective source file. -For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make sure to [avoid splitting sentences when adding links](#avoid-splitting-sentences-when-adding-links). +Placeholders in translated text should match the respective source file's code style. For example +use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make sure to +[avoid splitting sentences when adding links](#avoid-splitting-sentences-when-adding-links). - In Ruby/HAML: @@ -257,9 +258,9 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s Use the [`GlSprintf`](https://gitlab-org.gitlab.io/gitlab-ui/?path=/docs/utilities-sprintf--sentence-with-link) component if: - - you need to include child components in the translation string. - - you need to include HTML in your translation string. - - you are using `sprintf` and need to pass `false` as the third argument to + - You need to include child components in the translation string. + - You need to include HTML in your translation string. + - You're using `sprintf` and need to pass `false` as the third argument to prevent it from escaping placeholder values. For example: @@ -272,7 +273,7 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s ``` - In other cases it may be simpler to use `sprintf`, perhaps in a computed + In other cases, it might be simpler to use `sprintf`, perhaps in a computed property. For example: ```html @@ -344,7 +345,8 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s # => When size == 2: 'There are 2 mice.' ``` - Avoid using `%d` or count variables in singular strings. This allows more natural translation in some languages. + Avoid using `%d` or count variables in singular strings. This allows more natural translation in + some languages. - In JavaScript: @@ -363,13 +365,12 @@ For example use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make s The `n_` method should only be used to fetch pluralized translations of the same string, not to control the logic of showing different strings for different -quantities. Some languages have different quantities of target plural forms - -Chinese (simplified), for example, has only one target plural form in our -translation tool. This means the translator would have to choose to translate -only one of the strings and the translation would not behave as intended in the -other case. +quantities. Some languages have different quantities of target plural forms. +For example, Chinese (simplified) has only one target plural form in our +translation tool. This means the translator has to choose to translate only one +of the strings, and the translation doesn't behave as intended in the other case. -For example, prefer to use: +For example, use this: ```ruby if selected_projects.one? @@ -379,7 +380,7 @@ else end ``` -rather than: +Instead of this: ```ruby # incorrect usage example @@ -388,21 +389,22 @@ n_("%{project_name}", "%d projects selected", count) % { project_name: 'GitLab' ### Namespaces -A namespace is a way to group translations that belong together. They provide context to our translators by adding a prefix followed by the bar symbol (`|`). For example: +A namespace is a way to group translations that belong together. They provide context to our +translators by adding a prefix followed by the bar symbol (`|`). For example: ```ruby 'Namespace|Translated string' ``` -A namespace provide the following benefits: +A namespace: -- It addresses ambiguity in words, for example: `Promotions|Promote` vs `Epic|Promote` -- It allows translators to focus on translating externalized strings that belong to the same product area rather than arbitrary ones. -- It gives a linguistic context to help the translator. +- Addresses ambiguity in words. For example: `Promotions|Promote` vs `Epic|Promote`. +- Allows translators to focus on translating externalized strings that belong to the same product + area, rather than arbitrary ones. +- Gives a linguistic context to help the translator. -In some cases, namespaces don't make sense, for example, -for ubiquitous UI words and phrases such as "Cancel" or phrases like "Save changes" a namespace could -be counterproductive. +In some cases, namespaces don't make sense. For example, for ubiquitous UI words and phrases such as +"Cancel" or phrases like "Save changes," a namespace could be counterproductive. Namespaces should be PascalCase. @@ -412,7 +414,7 @@ Namespaces should be PascalCase. s_('OpenedNDaysAgo|Opened') ``` - In case the translation is not found it returns `Opened`. + If the translation isn't found, `Opened` is returned. - In JavaScript: @@ -420,18 +422,19 @@ Namespaces should be PascalCase. s__('OpenedNDaysAgo|Opened') ``` -The namespace should be removed from the translation. See the -[translation guidelines for more details](translation.md#namespaced-strings). +The namespace should be removed from the translation. For more details, see the +[translation guidelines](translation.md#namespaced-strings). ### HTML -We no longer include HTML directly in the strings that are submitted for translation. This is for a couple of reasons: +We no longer include HTML directly in the strings that are submitted for translation. This is +because: -1. It introduces a chance for the translated string to accidentally include invalid HTML. -1. It introduces a security risk where translated strings become an attack vector for XSS, as noted by the +1. The translated string can accidentally include invalid HTML. +1. Translated strings can become an attack vector for XSS, as noted by the [Open Web Application Security Project (OWASP)](https://owasp.org/www-community/attacks/xss/). -To include formatting in the translated string, we can do the following: +To include formatting in the translated string, you can do the following: - In Ruby/HAML: @@ -449,18 +452,18 @@ To include formatting in the translated string, we can do the following: // => 'Some bold text.' ``` -- In Vue +- In Vue: See the section on [interpolation](#interpolation). -When [this translation helper issue](https://gitlab.com/gitlab-org/gitlab/-/issues/217935) is complete, we plan to update the -process of including formatting in translated strings. +When [this translation helper issue](https://gitlab.com/gitlab-org/gitlab/-/issues/217935) +is complete, we plan to update the process of including formatting in translated strings. #### Including Angle Brackets -If a string contains angles brackets (`<`/`>`) that are not used for HTML, it is still flagged by the -`rake gettext:lint` linter. -To avoid this error, use the applicable HTML entity code (`<` or `>`) instead: +If a string contains angle brackets (`<`/`>`) that are not used for HTML, the `rake gettext:lint` +linter still flags it. To avoid this error, use the applicable HTML entity code (`<` or `>`) +instead: - In Ruby/HAML: @@ -493,12 +496,12 @@ To avoid this error, use the applicable HTML entity code (`<` or `>`) inst ### Numbers -Different locales may use different number formats. To support localization of numbers, we use `formatNumber`, -which leverages [`toLocaleString()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/toLocaleString). +Different locales may use different number formats. To support localization of numbers, we use +`formatNumber`, which leverages [`toLocaleString()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/toLocaleString). -`formatNumber` formats numbers as strings using the current user locale by default. +By default, `formatNumber` formats numbers as strings using the current user locale. -- In JavaScript +- In JavaScript: ```javascript import { formatNumber } from '~/locale'; @@ -509,7 +512,7 @@ const tenThousand = formatNumber(10000); // "10,000" (uses comma as decimal symb const fiftyPercent = formatNumber(0.5, { style: 'percent' }) // "50%" (other options are passed to toLocaleString) ``` -- In Vue templates +- In Vue templates: ```html