diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-17 18:19:40 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-17 18:19:40 +0000 |
| commit | ae845b62778b7c82aae1828b5f237e0468993ef8 (patch) | |
| tree | cdaa8a32f2fed3d8076fc01379cbad595a151331 | |
| parent | 79d6dbbd465b6bf0d4776fad62ec3e671f31b18f (diff) | |
| download | gitlab-ce-ae845b62778b7c82aae1828b5f237e0468993ef8.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
40 files changed, 1299 insertions, 823 deletions
diff --git a/.gitlab/issue_templates/Doc_cleanup.md b/.gitlab/issue_templates/Doc_cleanup.md new file mode 100644 index 00000000000..69caea1ae1e --- /dev/null +++ b/.gitlab/issue_templates/Doc_cleanup.md @@ -0,0 +1,53 @@ +<!-- +* Use this issue template for identifying issues to work on in existing documentation, normally identified +* with our [Vale](https://docs.gitlab.com/ee/development/documentation/testing.html#vale) or [markdownlint](https://docs.gitlab.com/ee/development/documentation/testing.html#markdownlint) tools. Much of this identified work is suitable for first-time contributors or +* for work during Hackathons. +* +* Normal documentation updates should use the Documentation template, and documentation work as part of +* feature development should use the Feature Request template. +--> + +## Identified documentation issue + +<!-- +* Include information about the issue that needs resolution. If the item is from an automated test, +* be sure to include a copy/paste from the the test results. [This issue](https://gitlab.com/gitlab-org/gitlab/-/issues/339543) is an example of text to include with a Vale issue. +* +* Limit the identified work to be related to one another, and keep it to a reasonable amount. For example, +* several moderate changes on one page, a few intermediate changes across five pages, or several very small +* changes for up to 10 pages. Larger items should be broken out into other issues to better distribute +* the opportunities for contributors. +--> + +## Process + +If you, as a contributor, decide to take this work on, assign this issue to yourself, and create one or more linked +merge requests that resolve this issue. Be sure to close this issue after all linked merge requests are completed. + +The work for this issue should involve only what's listed in the previous section. If you identify other work that +needs to be done, create separate, unlinked MRs as needed to address those items. + +When using automated test results for identified work, use this issue to work only on the listed lines. For +example, if the tests list several lines that show the word "admin" as needing to possibly be "administrator," +do not modify other parts of the page that may also include "admin," as the testing may have excluded those lines +(for example, they may be part of the **Admin Area** of GitLab). + +## Additional information + +<!-- +* Any concepts, procedures, reference info we could add to make it easier to successfully use GitLab? +* Include use cases, benefits, and/or goals for this work. +* If adding content: What audience is it intended for? (What roles and scenarios?) + For ideas, see personas at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ or the persona labels at + https://gitlab.com/groups/gitlab-org/-/labels?subscribed=&search=persona%3A +--> + +### Who can address the issue + +<!-- What if any special expertise is required to resolve this issue? --> + +### Other links/references + +<!-- For example, related GitLab issues/MRs --> + +/label ~documentation diff --git a/app/assets/javascripts/behaviors/shortcuts/shortcuts.js b/app/assets/javascripts/behaviors/shortcuts/shortcuts.js index 9297b14aac9..4d78c7b56a0 100644 --- a/app/assets/javascripts/behaviors/shortcuts/shortcuts.js +++ b/app/assets/javascripts/behaviors/shortcuts/shortcuts.js @@ -173,12 +173,7 @@ export default class Shortcuts { e.preventDefault(); const canaryCookieName = 'gitlab_canary'; const currentValue = parseBoolean(getCookie(canaryCookieName)); - setCookie(canaryCookieName, (!currentValue).toString(), { - expires: 365, - path: '/', - // next.gitlab.com uses a leading period. See https://gitlab.com/gitlab-org/gitlab/-/issues/350186 - domain: `.${window.location.hostname}`, - }); + setCookie(canaryCookieName, (!currentValue).toString(), { expires: 365, path: '/' }); refreshCurrentPage(); } diff --git a/app/assets/javascripts/environments/components/new_environment_item.vue b/app/assets/javascripts/environments/components/new_environment_item.vue index 27a763fb9c4..c039a663a68 100644 --- a/app/assets/javascripts/environments/components/new_environment_item.vue +++ b/app/assets/javascripts/environments/components/new_environment_item.vue @@ -40,6 +40,7 @@ export default { Terminal, TimeAgoTooltip, Delete, + EnvironmentAlert: () => import('ee_component/environments/components/environment_alert.vue'), }, directives: { GlTooltip, @@ -97,6 +98,9 @@ export default { hasDeployment() { return Boolean(this.environment?.upcomingDeployment || this.environment?.lastDeployment); }, + hasOpenedAlert() { + return this.environment?.hasOpenedAlert; + }, actions() { if (!this.lastDeployment) { return []; @@ -319,6 +323,9 @@ export default { class="gl-pl-4" /> </div> + <div v-if="hasOpenedAlert" class="gl-bg-gray-10 gl-md-px-7"> + <environment-alert :environment="environment" class="gl-pl-4 gl-py-5" /> + </div> </gl-collapse> </div> </template> diff --git a/app/assets/javascripts/environments/graphql/client.js b/app/assets/javascripts/environments/graphql/client.js index 64b18c2003b..26514b59995 100644 --- a/app/assets/javascripts/environments/graphql/client.js +++ b/app/assets/javascripts/environments/graphql/client.js @@ -2,6 +2,9 @@ import VueApollo from 'vue-apollo'; import createDefaultClient from '~/lib/graphql'; import environmentApp from './queries/environment_app.query.graphql'; import pageInfoQuery from './queries/page_info.query.graphql'; +import environmentToDeleteQuery from './queries/environment_to_delete.query.graphql'; +import environmentToRollbackQuery from './queries/environment_to_rollback.query.graphql'; +import environmentToStopQuery from './queries/environment_to_stop.query.graphql'; import { resolvers } from './resolvers'; import typeDefs from './typedefs.graphql'; @@ -33,6 +36,52 @@ export const apolloProvider = (endpoint) => { }, }, }); + + cache.writeQuery({ + query: environmentToDeleteQuery, + data: { + environmentToDelete: { + name: 'null', + __typename: 'LocalEnvironment', + id: '0', + deletePath: null, + folderPath: null, + retryUrl: null, + autoStopPath: null, + lastDeployment: null, + }, + }, + }); + cache.writeQuery({ + query: environmentToStopQuery, + data: { + environmentToStop: { + name: 'null', + __typename: 'LocalEnvironment', + id: '0', + deletePath: null, + folderPath: null, + retryUrl: null, + autoStopPath: null, + lastDeployment: null, + }, + }, + }); + cache.writeQuery({ + query: environmentToRollbackQuery, + data: { + environmentToRollback: { + name: 'null', + __typename: 'LocalEnvironment', + id: '0', + deletePath: null, + folderPath: null, + retryUrl: null, + autoStopPath: null, + lastDeployment: null, + }, + }, + }); return new VueApollo({ defaultClient, }); diff --git a/app/models/concerns/update_namespace_statistics.rb b/app/models/concerns/update_namespace_statistics.rb new file mode 100644 index 00000000000..26d6fc10228 --- /dev/null +++ b/app/models/concerns/update_namespace_statistics.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This module provides helpers for updating `NamespaceStatistics` with `after_save` and +# `after_destroy` hooks. +# +# Models including this module must respond to and return a `namespace` +# +# Example: +# +# class DependencyProxy::Manifest +# include UpdateNamespaceStatistics +# +# belongs_to :group +# alias_attribute :namespace, :group +# +# update_namespace_statistics namespace_statistics_name: :dependency_proxy_size +# end +module UpdateNamespaceStatistics + extend ActiveSupport::Concern + include AfterCommitQueue + + class_methods do + attr_reader :namespace_statistics_name, :statistic_attribute + + # Configure the model to update `namespace_statistics_name` on NamespaceStatistics, + # when `statistic_attribute` changes + # + # - namespace_statistics_name: A column of `NamespaceStatistics` to update + # - statistic_attribute: An attribute of the current model, default to `size` + def update_namespace_statistics(namespace_statistics_name:, statistic_attribute: :size) + @namespace_statistics_name = namespace_statistics_name + @statistic_attribute = statistic_attribute + + after_save(:schedule_namespace_statistics_refresh, if: :update_namespace_statistics?) + after_destroy(:schedule_namespace_statistics_refresh) + end + + private :update_namespace_statistics + end + + included do + private + + def update_namespace_statistics? + saved_change_to_attribute?(self.class.statistic_attribute) + end + + def schedule_namespace_statistics_refresh + run_after_commit do + Groups::UpdateStatisticsWorker.perform_async(namespace.id, [self.class.namespace_statistics_name]) + end + end + end +end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index f7b08f1d077..dc40ff62adb 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -5,8 +5,10 @@ class DependencyProxy::Blob < ApplicationRecord include TtlExpirable include Packages::Destructible include EachBatch + include UpdateNamespaceStatistics belongs_to :group + alias_attribute :namespace, :group MAX_FILE_SIZE = 5.gigabytes.freeze @@ -17,6 +19,7 @@ class DependencyProxy::Blob < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } mount_file_store_uploader DependencyProxy::FileUploader + update_namespace_statistics namespace_statistics_name: :dependency_proxy_size def self.total_size sum(:size) diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index c2587ffac9d..5ad746e4cd1 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -5,8 +5,10 @@ class DependencyProxy::Manifest < ApplicationRecord include TtlExpirable include Packages::Destructible include EachBatch + include UpdateNamespaceStatistics belongs_to :group + alias_attribute :namespace, :group MAX_FILE_SIZE = 10.megabytes.freeze DIGEST_HEADER = 'Docker-Content-Digest' @@ -20,6 +22,7 @@ class DependencyProxy::Manifest < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } mount_file_store_uploader DependencyProxy::FileUploader + update_namespace_statistics namespace_statistics_name: :dependency_proxy_size def self.find_by_file_name_or_digest(file_name:, digest:) find_by(file_name: file_name) || find_by(digest: digest) diff --git a/app/presenters/ci/pipeline_presenter.rb b/app/presenters/ci/pipeline_presenter.rb index 2818e6da036..410b633df50 100644 --- a/app/presenters/ci/pipeline_presenter.rb +++ b/app/presenters/ci/pipeline_presenter.rb @@ -5,7 +5,6 @@ module Ci include Gitlab::Utils::StrongMemoize delegator_override_with Gitlab::Utils::StrongMemoize # This module inclusion is expected. See https://gitlab.com/gitlab-org/gitlab/-/issues/352884. - delegator_override_with ActionView::Helpers::TagHelper # TODO: Remove `ActionView::Helpers::UrlHelper` inclusion as it overrides `Ci::Pipeline#tag` # We use a class method here instead of a constant, allowing EE to redefine # the returned `Hash` more easily. diff --git a/config/feature_flags/development/merge_service_ping_instrumented_metrics.yml b/config/feature_flags/development/merge_service_ping_instrumented_metrics.yml new file mode 100644 index 00000000000..e108ab98bd7 --- /dev/null +++ b/config/feature_flags/development/merge_service_ping_instrumented_metrics.yml @@ -0,0 +1,8 @@ +--- +name: merge_service_ping_instrumented_metrics +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77629 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/352964 +milestone: '14.8' +type: development +group: group::product intelligence +default_enabled: false diff --git a/doc/topics/release_your_application.md b/doc/topics/release_your_application.md index 64decba33ad..8fd4606c375 100644 --- a/doc/topics/release_your_application.md +++ b/doc/topics/release_your_application.md @@ -31,7 +31,7 @@ to Kubernetes clusters using the [GitLab Agent](../user/clusters/agent/install/i #### GitOps deployments **(PREMIUM)** With the [GitLab Agent](../user/clusters/agent/install/index.md), you can perform [pull-based -deployments of Kubernetes manifests](../user/clusters/agent/repository.md#synchronize-manifest-projects). This provides a scalable, secure, and cloud-native +deployments of Kubernetes manifests](../user/clusters/agent/gitops.md). This provides a scalable, secure, and cloud-native approach to manage Kubernetes deployments. #### Deploy to Kubernetes with the CI/CD Tunnel diff --git a/doc/user/application_security/cluster_image_scanning/index.md b/doc/user/application_security/cluster_image_scanning/index.md index 0db9af7a0d3..8b29da06b57 100644 --- a/doc/user/application_security/cluster_image_scanning/index.md +++ b/doc/user/application_security/cluster_image_scanning/index.md @@ -292,7 +292,7 @@ scan images from within your Kubernetes cluster and record the vulnerabilities i ### Configuration The Agent runs the cluster image scanning once the `cluster_image_scanning` -directive is added to your [Agent's configuration repository](../../clusters/agent/repository.md#scan-your-container-images-for-vulnerabilities). +directive is added to your [Agent's configuration repository](../../clusters/agent/vulnerabilities.md). ## Security Dashboard @@ -302,7 +302,7 @@ the security vulnerabilities in your groups, projects, and pipelines. ## Interacting with the vulnerabilities After you find a vulnerability, you can address it in the [vulnerability report](../vulnerabilities/index.md) -or the [GitLab Agent's](../../clusters/agent/install/index.md#view-vulnerabilities-in-cluster-images) +or the [GitLab Agent's](../../clusters/agent/vulnerabilities.md) details section. ## Troubleshooting diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index 7857de8c780..6ce25984c64 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -13,7 +13,7 @@ job is fully executed. ## Scan result policy editor -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77814) in GitLab 14.8 with a flag named `scan_result_policy`. Disabled by default. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77814) in GitLab 14.8. NOTE: Only project Owners have the [permissions](../../permissions.md#project-members-permissions) diff --git a/doc/user/clusters/agent/ci_cd_tunnel.md b/doc/user/clusters/agent/ci_cd_tunnel.md index 5fe772d9686..62b634ea8ff 100644 --- a/doc/user/clusters/agent/ci_cd_tunnel.md +++ b/doc/user/clusters/agent/ci_cd_tunnel.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# CI/CD Tunnel **(FREE)** +# Using a GitLab CI/CD workflow for Kubernetes **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327409) in GitLab 14.1. > - The pre-configured `KUBECONFIG` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324275) in GitLab 14.2. @@ -12,52 +12,215 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) to GitLab Free in 14.5. > - Support for Omnibus installations was [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5686) in GitLab 14.5. -To use GitLab CI/CD to safely deploy your application to a cluster, you can use the CI/CD Tunnel. +You can use GitLab CI/CD to safely deploy to and update your Kubernetes clusters. -You can authorize multiple projects to access the same cluster, so you -can keep your application's codebase in one repository and configure -your cluster in another. This method is scalable and can save you resources. +To do so, you install a GitLab agent in your cluster. Then in your GitLab CI/CD pipelines, +you can refer to the cluster connection as a Kubernetes context. +Then you can run Kubernetes API commands as part of your GitLab CI/CD pipeline. -To ensure access to your cluster is safe, only the projects you -authorize can access your Agent through the CI/CD Tunnel. +To ensure access to your cluster is safe: -## Prerequisites +- Each agent has a separate context (`kubecontext`). +- Only the project where the agent is, and any additional projects you authorize can access the agent in your cluster. -To use the CI/CD Tunnel, you need an existing Kubernetes cluster connected to GitLab through the -[GitLab Agent](install/index.md#install-the-agent-onto-the-cluster). +You do not need to have a runner in the cluster with the agent. -To run your CI/CD jobs using the CI/CD Tunnel, you do not need to have a runner in the same cluster. +## GitLab CI/CD workflow steps -## How the CI/CD Tunnel works +To update a Kubernetes cluster by using GitLab CI/CD, complete the following steps. -When you authorize a project to use an Agent, the Tunnel automatically -injects a `KUBECONFIG` variable into its CI/CD jobs. This way, you can -run `kubectl` commands from GitLab CI/CD scripts that belong to the -authorized project. +1. Ensure you have a working Kubernetes cluster and the manifests are in a GitLab project. +1. In the same GitLab project, [register and install the GitLab agent](install/index.md). +1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to + select the agent's Kubernetes context and run the Kubernetes API commands. +1. Run your pipeline to deploy to or update the cluster. -When you authorize a group, all the projects that belong to that group -become authorized to access the selected Agent. +If you have multiple GitLab projects that contain Kubernetes manifests: -An Agent can only authorize projects or groups in the same group -hierarchy as the Agent's configuration project. You can authorize -up to 100 projects and 100 groups per Agent. +1. [Install the GitLab agent](install/index.md) in its own project, or in one of the + GitLab projects where you keep Kubernetes manifests. +1. [Authorize the agent](#authorize-the-agent) to access your GitLab projects. +1. Optional. For added security, [use impersonation](#use-impersonation-to-restrict-project-and-group-access). +1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to + select the agent's Kubernetes context and run the Kubernetes API commands. +1. Run your pipeline to deploy to or update the cluster. -Also, each Agent has a separate context (`kubecontext`). -The Tunnel uses this information to safely allow access to the cluster from -jobs running in the projects you authorized. +## Authorize the agent -### `~/.kube/cache` permissions +You must authorize the agent to access the project where you keep your Kubernetes manifests. +You can authorize the agent to access individual projects, or authorize a group or subgroup, +so all projects within have access. For added security, you can also +[use impersonation](#use-impersonation-to-restrict-project-and-group-access). + +### Authorize the agent to access your projects + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327850) in GitLab 14.4. + +To authorize the agent to access the GitLab project where you keep Kubernetes manifests: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). +1. Edit the file. Under the `ci_access` keyword, add the `projects` attribute. +1. For the `id`, add the path: + + ```yaml + ci_access: + projects: + - id: path/to/project + ``` + + - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. + - You can authorize up to 100 projects. + +All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. +Choose the context to run `kubectl` commands from your CI/CD scripts. + +### Authorize the agent to access projects in your groups + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. + +To authorize the agent to access all of the GitLab projects in a group or subgroup: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). +1. Edit the file. Under the `ci_access` keyword, add the `groups` attribute. +1. For the `id`, add the path: + + ```yaml + ci_access: + groups: + - id: path/to/group/subgroup + ``` + + - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. + - You can authorize up to 100 groups. + +All the projects that belong to the group are now authorized to access the agent. +All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. +Choose the context to run `kubectl` commands from your CI/CD scripts. + +## Update your `.gitlab-ci.yml` file to run `kubectl` commands + +In the project where you want to run Kubernetes commands, edit your project's `.gitlab-ci.yml` file. + +In the first command under the `script` keyword, set your agent's context. +Use the format `path/to/agent/repository:agent-name`. For example: + +```yaml + deploy: + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config get-contexts + - kubectl config use-context path/to/agent/repository:agent-name + - kubectl get pods +``` + +If you are not sure what your agent's context is, open a terminal and connect to your cluster. +Run `kubectl config get-contexts`. + +## Use impersonation to restrict project and group access **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345014) in GitLab 14.5. + +By default, your CI/CD job inherits all the permissions from the service account used to install the +agent in the cluster. +To restrict access to your cluster, you can use [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). + +To specify impersonations, use the `access_as` attribute in your agent configuration file and use Kubernetes RBAC rules to manage impersonated account permissions. + +You can impersonate: + +- The agent itself (default). +- The CI/CD job that accesses the cluster. +- A specific user or system account defined within the cluster. + +### Impersonate the agent + +The agent is impersonated by default. You don't need to do anything to impersonate it. + +### Impersonate the CI/CD job that accesses the cluster + +To impersonate the CI/CD job that accesses the cluster, under the `access_as` key, add the `ci_job: {}` key-value. + +When the agent makes the request to the actual Kubernetes API, it sets the +impersonation credentials in the following way: + +- `UserName` is set to `gitlab:ci_job:<job id>`. Example: `gitlab:ci_job:1074499489`. +- `Groups` is set to: + - `gitlab:ci_job` to identify all requests coming from CI jobs. + - The list of IDs of groups the project is in. + - The project ID. + - The slug of the environment this job belongs to. + + Example: for a CI job in `group1/group1-1/project1` where: + + - Group `group1` has ID 23. + - Group `group1/group1-1` has ID 25. + - Project `group1/group1-1/project1` has ID 150. + - Job running in a prod environment. + + Group list would be `[gitlab:ci_job, gitlab:group:23, gitlab:group:25, gitlab:project:150, gitlab:project_env:150:prod]`. + +- `Extra` carries extra information about the request. The following properties are set on the impersonated identity: + +| Property | Description | +| -------- | ----------- | +| `agent.gitlab.com/id` | Contains the agent ID. | +| `agent.gitlab.com/config_project_id` | Contains the agent's configuration project ID. | +| `agent.gitlab.com/project_id` | Contains the CI project ID. | +| `agent.gitlab.com/ci_pipeline_id` | Contains the CI pipeline ID. | +| `agent.gitlab.com/ci_job_id` | Contains the CI job ID. | +| `agent.gitlab.com/username` | Contains the username of the user the CI job is running as. | +| `agent.gitlab.com/environment_slug` | Contains the slug of the environment. Only set if running in an environment. | + +Example to restrict access by the CI/CD job's identity: + +```yaml +ci_access: + projects: + - id: path/to/project + access_as: + ci_job: {} +``` + +### Impersonate a static identity + +For a given connection, you can use a static identity for the impersonation. + +Under the `access_as` key, add the `impersonate` key to make the request using the provided identity. + +The identity can be specified with the following keys: + +- `username` (required) +- `uid` +- `groups` +- `extra` + +See the [official Kubernetes documentation for details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). + +## Troubleshooting + +### `kubectl` commands not supported + +The commands `kubectl exec`, `kubectl cp`, and `kubectl attach` are not supported. +Anything that uses these API endpoints does not work, because they use the deprecated +SPDY protocol. +[An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/346248) to add support for these commands. + +### Grant write permissions to `~/.kube/cache` -`kubectl` and other tools based on the same libraries (such as Helm, `kpt`, and `kustomize`) cache information about +Tools like `kubectl`, Helm, `kpt`, and `kustomize` cache information about the cluster in `~/.kube/cache`. If this directory is not writable, the tool fetches information on each invocation, -making interactions slower and creating unnecessary load on the cluster. Make sure that this directory in the container image -you use is writable for the best experience. +making interactions slower and creating unnecessary load on the cluster. For the best experience, in the +image you use in your .`gitlab-ci.yml` file, ensure this directory is writable. + +### Enable TLS -## Configure the CI/CD Tunnel +If you are on a self-managed GitLab instance, ensure your instance is configured with Transport Layer Security (TLS). -The CI/CD Tunnel is configured directly through the -Agent's configuration file ([`config.yaml`](repository.md)) to: +If you attempt to use `kubectl` without TLS, you might get an error like: -- Authorize [projects](repository.md#authorize-projects-to-use-an-agent) and [groups](repository.md#authorize-groups-to-use-an-agent) to use the same Agent. -- [Run `kubectl` commands using the CI/CD Tunnel](repository.md#run-kubectl-commands-using-the-cicd-tunnel). -- [Restrict access of authorized projects and groups through impersonation strategies](repository.md#use-impersonation-to-restrict-project-and-group-access). +```shell +$ kubectl get pods +error: You must be logged in to the server (the server has asked for the client to provide credentials) +``` diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md new file mode 100644 index 00000000000..c527747b605 --- /dev/null +++ b/doc/user/clusters/agent/gitops.md @@ -0,0 +1,155 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Using a GitOps workflow for Kubernetes **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. + +With GitOps, you can manage containerized clusters and applications from a Git repository that: + +- Is the single source of truth of your system. +- Is the single place where you operate your system. + +By combining GitLab, Kubernetes, and GitOps, you can have: + +- GitLab as the GitOps operator. +- Kubernetes as the automation and convergence system. +- GitLab CI/CD for Continuous Integration and the agent for Continuous Deployment. + +This diagram shows the repositories and main actors in a GitOps deployment: + +```mermaid +sequenceDiagram + participant D as Developer + participant A as Application code repository + participant M as Manifest repository + participant K as GitLab agent + participant C as Agent configuration repository + loop Regularly + K-->>C: Grab the configuration + end + D->>+A: Pushing code changes + A->>M: Updating manifest + loop Regularly + K-->>M: Watching changes + M-->>K: Pulling and applying changes + end +``` + +For details, view the [architecture documentation](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md#high-level-architecture). + +## GitOps workflow steps + +To update a Kubernetes cluster by using GitOps, complete the following steps. + +1. Ensure you have a working Kubernetes cluster, and that the manifests are in a GitLab project. +1. In the same project, [register and install the GitLab agent](install/index.md). +1. Configure the agent configuration file so that the agent monitors the project for changes to the Kubernetes manifests. + Use the [GitOps configuration reference](#gitops-configuration-reference) for guidance. + +Any time you commit updates to your Kubernetes manifests, the agent updates the cluster. + +### GitOps configuration reference + +The following snippet shows the possible keys and values for the GitOps section of an agent configuration file. + +```yaml +gitops: + # The agent watches projects where your manifests are stored. + # When a project changes, the agent deploys the changes to the cluster. + manifest_projects: + # No authentication mechanisms are currently supported. + # The `id` is a path to a Git repository that has + # Kubernetes manifests in YAML or JSON format. + - id: gitlab-org/cluster-integration/gitlab-agent + # Namespace to use if not set explicitly in object manifest. + # Also used for inventory ConfigMap objects. + default_namespace: my-ns + # Paths in the repository you want to scan for manifest files. + # Directories with names that start with a dot are ignored. + paths: + # Read all .yaml files from team1/app1 directory. + # See https://github.com/bmatcuk/doublestar#about and + # https://pkg.go.dev/github.com/bmatcuk/doublestar/v2#Match for globbing rules. + - glob: '/team1/app1/*.yaml' + # Read all .yaml files from team2/apps and all subdirectories. + - glob: '/team2/apps/**/*.yaml' + # If 'paths' is not specified or is an empty list, the configuration below is used. + - glob: '/**/*.{yaml,yml,json}' + # Reconcile timeout defines whether the applier should wait + # until all applied resources have been reconciled, and if so, + # how long to wait. + reconcile_timeout: 3600s # 1 hour by default + # Dry run strategy defines whether changes should actually be performed, + # or if it is just talk and no action. + # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/common/common.go#L68-L89 + # Can be: none, client, server + dry_run_strategy: none # 'none' by default + # Prune defines whether pruning of previously applied + # objects should happen after apply. + prune: true # enabled by default + # Prune timeout defines whether we should wait for all resources + # to be fully deleted after pruning, and if so, how long we should + # wait. + prune_timeout: 3600s # 1 hour by default + # Prune propagation policy defines the deletion propagation policy + # that should be used for pruning. + # https://github.com/kubernetes/apimachinery/blob/44113beed5d39f1b261a12ec398a356e02358307/pkg/apis/meta/v1/types.go#L456-L470 + # Can be: orphan, background, foreground + prune_propagation_policy: foreground # 'foreground' by default + # Inventory policy defines if an inventory object can take over + # objects that belong to another inventory object or don't + # belong to any inventory object. + # This is done by determining if the apply/prune operation + # can go through for a resource based on comparison of + # the inventory-id value in the package and the owning-inventory + # annotation (config.k8s.io/owning-inventory) in the live object. + # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/inventory/policy.go#L12-L66 + # Can be: must_match, adopt_if_no_inventory, adopt_all + inventory_policy: must_match # 'must_match' by default +``` + +## Troubleshooting + +### Avoiding conflicts when you have multiple projects + +The agent watches each glob pattern set under a project's `paths` section independently, and makes updates to the cluster concurrently. +If changes are found at multiple paths, when the agent attempts to update the cluster, +a conflict can occur. + +To prevent this from happening, consider storing a logical group of manifests in a single place and reference them only once to avoid overlapping globs. + +For example, both of these globs match `*.yaml` files in the root directory +and could cause conflicts: + +```yaml +gitops: + manifest_projects: + - id: project1 + paths: + - glob: '/**/*.yaml' + - glob: '/*.yaml' +``` + +Instead, specify a single glob that matches all `*.yaml` files recursively: + +```yaml +gitops: + manifest_projects: + - id: project1 + paths: + - glob: '/**/*.yaml' +``` + +### Use multiple agents or projects + +If you store your Kubernetes manifests in separate GitLab projects, +update your agent configuration file with the location of these projects. + +WARNING: +The project with the agent's +configuration file can be private or public. Other projects with Kubernetes manifests must be public. Support for private manifest projects is tracked +in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/283885). diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index 3fb141e402f..7d14b495c87 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -16,11 +16,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can connect your Kubernetes cluster with GitLab to deploy, manage, and monitor your cloud-native solutions. You can choose from two primary workflows. -In a **GitOps workflow**, you keep your Kubernetes manifests in GitLab. You install a GitLab agent in your cluster, and +In a [**GitOps** workflow](gitops.md), you keep your Kubernetes manifests in GitLab. You install a GitLab agent in your cluster, and any time you update your manifests, the agent updates the cluster. This workflow is fully driven with Git and is considered pull-based, because the cluster is pulling updates from your GitLab repository. -In a **CI/CD** workflow, you use GitLab CI/CD to query and update your cluster by using the Kubernetes API. +In a [**CI/CD** workflow](ci_cd_tunnel.md), you use GitLab CI/CD to query and update your cluster by using the Kubernetes API. This workflow is considered push-based, because GitLab is pushing requests from GitLab CI/CD to your cluster. Both of these workflows require you to [install an agent in your cluster](install/index.md). @@ -47,174 +47,15 @@ version. The list of supported versions is based on: Some GitLab features might work on versions not listed here. -## Using Kubernetes with GitOps **(PREMIUM)** +## Migrate to the agent from the legacy certificate-based integration -With GitOps, you can manage containerized clusters and applications from a Git repository that: - -- Is the single source of truth of your system. -- Is the single place where you operate your system. - -By combining GitLab, Kubernetes, and GitOps, you can have: - -- GitLab as the GitOps operator. -- Kubernetes as the automation and convergence system. -- GitLab CI/CD for Continuous Integration and the agent for Continuous Deployment. - -Beyond that, you can use all the features offered by GitLab as -the all-in-one DevOps platform for your product and your team. - -### GitOps workflow **(PREMIUM)** - -The agent uses multiple GitLab projects to provide a flexible workflow -that can suit various needs. This diagram shows these repositories and the main -The agent uses multiple GitLab projects to provide a flexible workflow. -This diagram shows these repositories and the main -actors involved in a deployment: - -```mermaid -sequenceDiagram - participant D as Developer - participant A as Application code repository - participant M as Manifest repository - participant K as GitLab agent - participant C as Agent configuration repository - loop Regularly - K-->>C: Grab the configuration - end - D->>+A: Pushing code changes - A->>M: Updating manifest - loop Regularly - K-->>M: Watching changes - M-->>K: Pulling and applying changes - end -``` - -For details, view the [architecture documentation](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md#high-level-architecture). - -To perform GitOps deployments, you need: - -- A properly-configured Kubernetes cluster where the GitLab agent is running. -- A project that contains the agent's configuration file (`config.yaml`) in the repository. - This file tells the agent which repositories to synchronize with the cluster. -- A project that contains Kubernetes manifests. Any changes to manifests are applied to the cluster. - -You can keep the agent's configuration file and Kubernetes manifests in one project, or you can use multiple. - -- One GitLab project (recommended): When you use one project for both the Kubernetes manifests - and the agent's configuration file, the projects can be either private or public. -- Two GitLab projects: When you use two different GitLab projects (one for Kubernetes - manifests and another for the agent's configuration file), the project with Kubernetes manifests must - be public. The project with the agent's configuration file can be either private or public. - -Support for separate private projects is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/283885). - -## Remove an agent - -You can remove an agent by using the [GitLab UI](#remove-an-agent-through-the-gitlab-ui) or the [GraphQL API](#remove-an-agent-with-the-gitlab-graphql-api). - -### Remove an agent through the GitLab UI - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323055) in GitLab 14.7. - -To remove an agent from the UI: - -1. On the top bar, select **Menu > Projects** and find the project that contains the agent's configuration file. -1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. In the table, in the row for your agent, in the **Options** column, select the vertical ellipsis (**{ellipsis_v}**). -1. Select **Delete agent**. - -### Remove an agent with the GitLab GraphQL API - -1. Get the `<cluster-agent-token-id>` from a query in the interactive GraphQL explorer. - - For GitLab.com, go to <https://gitlab.com/-/graphql-explorer> to open GraphQL Explorer. - - For self-managed GitLab, go to `https://gitlab.example.com/-/graphql-explorer`, replacing `gitlab.example.com` with your instance's URL. - - ```graphql - query{ - project(fullPath: "<full-path-to-agent-configuration-project>") { - clusterAgent(name: "<agent-name>") { - id - tokens { - edges { - node { - id - } - } - } - } - } - } - ``` - -1. Remove an agent record with GraphQL by deleting the `clusterAgentToken`. - - ```graphql - mutation deleteAgent { - clusterAgentDelete(input: { id: "<cluster-agent-id>" } ) { - errors - } - } - - mutation deleteToken { - clusterAgentTokenDelete(input: { id: "<cluster-agent-token-id>" }) { - errors - } - } - ``` - -1. Verify whether the removal occurred successfully. If the output in the Pod logs includes `unauthenticated`, it means that the agent was successfully removed: - - ```json - { - "level": "warn", - "time": "2021-04-29T23:44:07.598Z", - "msg": "GetConfiguration.Recv failed", - "error": "rpc error: code = Unauthenticated desc = unauthenticated" - } - ``` - -1. Delete the agent in your cluster: - - ```shell - kubectl delete -n gitlab-kubernetes-agent -f ./resources.yml - ``` - -## Migrating to the agent from the legacy certificate-based integration - -Find out how to [migrate to the agent for Kubernetes](../../infrastructure/clusters/migrate_to_gitlab_agent.md) from the certificate-based integration. - -## Kubernetes network security alerts **(ULTIMATE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Cilium integration is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -for use in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The agent for Kubernetes also provides an integration with Cilium. This integration provides a simple way to -generate network policy-related alerts and to surface those alerts in GitLab. - -Several components work in concert for the agent to generate the alerts: - -- A working Kubernetes cluster. -- Cilium integration through either of these options: - - Installation through [cluster management template](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). - - Enablement of [hubble-relay](https://docs.cilium.io/en/v1.8/concepts/overview/#hubble) on an - existing installation. -- One or more network policies through any of these options: - - Use the [Container Network Policy editor](../../application_security/policies/index.md#container-network-policy-editor) to create and manage policies. - - Use an [AutoDevOps](../../application_security/policies/index.md#container-network-policy) configuration. - - Add the required labels and annotations to existing network policies. -- A configuration repository with [Cilium configured in `config.yaml`](repository.md#surface-network-security-alerts-from-cluster-to-gitlab) - -The setup process follows the same [agent's installation steps](install/index.md), -with the following differences: - -- When you define a configuration repository, you must do so with [Cilium settings](repository.md#surface-network-security-alerts-from-cluster-to-gitlab). -- You do not need to specify the `gitops` configuration section. +Read about how to [migrate to the agent for Kubernetes](../../infrastructure/clusters/migrate_to_gitlab_agent.md) from the certificate-based integration. ## Related topics +- [GitOps workflow](gitops.md) +- [GitLab CI/CD workflow](ci_cd_tunnel.md) +- [Install the agent](install/index.md) +- [Work with the agent](repository.md) - [Troubleshooting](troubleshooting.md) -- [Contribute to the GitLab agent's development](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/doc) +- [Contribute to the agent's development](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/doc) diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index 4d196e57f8f..fd51b816124 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -4,92 +4,85 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Install the GitLab Agent **(FREE)** +# Installing the agent for Kubernetes **(FREE)** > - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5. > - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/merge_requests/594) multi-arch images in GitLab 14.8. The first multi-arch release is `v14.8.1`. It supports AMD64 and ARM64 architectures. -To connect a cluster to GitLab, you need to install the GitLab Agent -onto your cluster. +To connect a Kubernetes cluster to GitLab, you must install an agent in your cluster. ## Prerequisites -- An existing Kubernetes cluster. If you don't have a cluster yet, you can create a new cluster on cloud providers, such as: +Before you can install the agent in your cluster, you need: + +- An existing Kubernetes cluster. If you don't have a cluster, you can create one on a cloud provider, like: - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/quickstart) - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) -- On self-managed GitLab instances, a GitLab administrator needs to set up the [GitLab Agent Server (KAS)](../../../../administration/clusters/kas.md). +- On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). ## Installation steps -To install the GitLab Agent on your cluster: - -1. [Define a configuration repository](#define-a-configuration-repository). -1. [Register the Agent with GitLab](#register-the-agent-with-gitlab). -1. [Install the Agent onto the cluster](#install-the-agent-onto-the-cluster). +To install the agent in your cluster: -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a GitLab 14.2 [walking-through video](https://www.youtube.com/watch?v=XuBpKtsgGkE) with this process. +1. [Create an agent configuration file called `config.yaml`](#create-an-agent-configuration-file). +1. [Register the agent with GitLab](#register-the-agent-with-gitlab). +1. [Install the agent in your cluster](#install-the-agent-in-the-cluster). -When you complete the installation process, you can -[view your Agent's status and activity information](#view-your-agents). -You can also [configure](#configure-the-agent) it to your needs. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a GitLab 14.2 [walk-through of this process](https://www.youtube.com/watch?v=XuBpKtsgGkE). -### Define a configuration repository +### Create an agent configuration file -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the Agent manifest configuration can be added to multiple directories (or subdirectories) of its repository. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. > - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. -To create an Agent, you need a GitLab repository to hold its -configuration file. If you already have a repository holding your -cluster's manifest files, you can use it to store your -Agent's configuration file and sync them with no further steps. - -#### Create the Agent's configuration file - -To create an Agent, go to the repository where you want to store -it and add the Agent's configuration file under: +In a GitLab project, in the repository, create a file called `config.yaml` at this path: ```plaintext .gitlab/agents/<agent-name>/config.yaml ``` -You **don't have to add any content** to this file at the moment you -create it. The fact that the file exists tells GitLab that this is -an Agent. You can edit it later to [configure the Agent](#configure-the-agent). - -When creating this file, pay special attention to: +- Ensure the agent name follows the [naming convention](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name). +- Ensure the filename has the `.yaml` file extension (`config.yaml`). The `.yml` extension is not accepted. +- Add content to the `config.yaml` file: + - For a GitOps workflow, view [the configuration reference](../gitops.md#gitops-configuration-reference) for details. + - For a GitLab CI/CD workflow, you can leave the file blank for now. -- The [Agent's naming format](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name). -- The file extension: use the `.yaml` extension (`config.yaml`). The `.yml` extension is **not** recognized. +The agent bootstraps with the GitLab installation URL and an authentication token, +and you provide the rest of the configuration in your repository, following +Infrastructure as Code (IaaC) best practices. -### Register the Agent with GitLab +### Register the agent with GitLab -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5786) in GitLab 14.1, you can create a new Agent record directly from the GitLab UI. +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5786) in GitLab 14.1, you can create a new agent record directly from the GitLab UI. -Now that you've created your Agent's configuration file, register it +Now that you've created your agent configuration file, register it with GitLab. -When you register the Agent, GitLab generates a token that you need for -installing the Agent onto your cluster. +When you register the agent, GitLab generates a token that you need to +install the agent in your cluster. + +Prerequisite when using a [GitLab CI/CD workflow](../ci_cd_tunnel.md): -In GitLab, go to the project where you added your Agent's configuration -file and: +- In the project that has the agent configuration file, ensure that [GitLab CI/CD is enabled](../../../../ci/enable_or_disable_ci.md#enable-cicd-in-a-project). -1. Ensure that [GitLab CI/CD is enabled in your project](../../../../ci/enable_or_disable_ci.md#enable-cicd-in-a-project). -1. From your project's sidebar, select **Infrastructure > Kubernetes clusters**. +To register the agent with GitLab: + +1. On the top bar, select **Menu > Projects** and find the project that has your agent configuration file. +1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. 1. Select **Actions**. -1. From the **Select an Agent** dropdown list, select the Agent you want to register and select **Register an Agent**. -1. GitLab generates a registration token for this Agent. Securely store this secret token, as you need it to install the Agent onto your cluster and to [update the Agent](#update-the-agent-version) to another version. -1. Copy the command under **Recommended installation method**. You need it to install the Agent onto your cluster through the one-liner installation method. +1. From the **Select an agent** dropdown list, select the agent you want to register and select **Register an agent**. +1. GitLab generates a registration token for this agent. Securely store this secret token. You need it to install the agent in your cluster and to [update the agent](#update-the-agent-version) to another version. +1. Copy the command under **Recommended installation method**. You need it when you use the one-liner installation method to install the agent in your cluster. -### Install the Agent onto the cluster +### Install the agent in the cluster -To connect your cluster to GitLab, install the registered Agent -onto your cluster. To install it, you can use either: +To connect your cluster to GitLab, install the registered agent +in your cluster. To install it, you can use either: - [The one-liner installation method](#one-liner-installation). - [The advanced installation method](#advanced-installation). -You can use the one-liner installation for trying to use the Agent for the first time, to do internal setups with +You can use the one-liner installation for trying to use the agent for the first time, to do internal setups with high trust, and to quickly get started. For long-term production usage, you may want to use the advanced installation method to benefit from more configuration options. @@ -99,35 +92,33 @@ The one-liner installation is the simplest process, but you need Docker installed locally. If you don't have it, you can either install it or opt to the [advanced installation method](#advanced-installation). -To install the Agent on your cluster using the one-liner installation: +To install the agent on your cluster using the one-liner installation: -1. In your computer, open the terminal and connect to your cluster. +1. In your computer, open a terminal and [connect to your cluster](https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/). 1. Run the command you copied when registering your cluster in the previous step. Optionally, you can [customize the one-liner installation command](#customize-the-one-liner-installation). ##### Customize the one-liner installation -The one-liner command generated by GitLab: +By default, the one-liner command generated by GitLab: - Creates a namespace for the deployment (`gitlab-kubernetes-agent`). - Sets up a service account with `cluster-admin` rights (see [how to restrict this service account](#customize-the-permissions-for-the-agentk-service-account)). -- Creates a `Secret` resource for the Agent's registration token. +- Creates a `Secret` resource for the agent's registration token. - Creates a `Deployment` resource for the `agentk` pod. -You can edit these parameters according to your needs to customize the -one-liner installation command at the command line. To find all available -options, run in your terminal: +You can edit these parameters to customize the one-liner installation command. +To view all available options, open a terminal and run this command: ```shell docker run --pull=always --rm registry.gitlab.com/gitlab-org/cluster-integration/gitlab-agent/cli:stable generate --help ``` WARNING: -`--agent-version stable` can be used to refer to the latest stable -release at the time when the command runs. It's fine for testing -purposes but for production please make sure to specify a matching -version explicitly. +Use `--agent-version stable` to refer to the latest stable +release at the time when the command runs. For production, however, +you should explicitly specify a matching version. #### Advanced installation @@ -135,105 +126,76 @@ For advanced installation options, use [the `kpt` installation method](https://g ##### Customize the permissions for the `agentk` service account -The GitLab Agent allows you to fully own your cluster and grant GitLab -the permissions you want. Still, to facilitate the process, by default the -generated manifests provide `cluster-admin` rights to the Agent. +You own your cluster and can grant GitLab the permissions you want. +By default, however, the generated manifests provide `cluster-admin` rights to the agent. -You can restrict the Agent's access rights using Kustomize overlays. [An example is commented out](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/build/deployment/gitlab-agent/cluster/kustomization.yaml) in the `kpt` package you retrieved as part of the installation. +You can restrict the agent's access rights by using Kustomize overlays. [An example is commented out](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/build/deployment/gitlab-agent/cluster/kustomization.yaml) in the `kpt` package you retrieved as part of the installation. -To create restricted permissions: +To restrict permissions: 1. Copy the `cluster` directory. 1. Edit the `kustomization.yaml` and `components/*` files based on your requirements. 1. Run `kustomize build <your copied directory> | kubectl apply -f -` to apply your configuration. -The above setup allows you to regularly update from the upstream package using `kpt pkg update gitlab-agent --strategy resource-merge` and maintain your customizations at the same time. - -## Configure the Agent - -When successfully installed, you can [configure the Agent](../repository.md) -by editing its configuration file. -When you update the configuration file, GitLab transmits the -information to the cluster automatically without downtime. - -## View your Agents - -> The version of installed `agentk` shown on the Agent tab [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8. -If you have at least the Developer role, you can access the Agent's -configuration repository and view the Agent's list: +#### Update the advanced installation base layer -1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. Select **Agent** tab to view clusters connected to GitLab through the Agent. +Now you can update from the upstream package by using `kpt pkg update gitlab-agent --strategy resource-merge`. +When the advanced installation setup changes, you will not need to change your custom overlays. -On this page, you can view: +## Install multiple agents in your cluster -- All the registered Agents for the current project. -- The connection status. -- The version of `agentk` installed on your cluster. -- The path to each Agent's configuration file. +For total separation between teams, you might need to run multiple `agentk` instances in your cluster. +You might want multiple agents so you can restrict RBAC for every `agentk` deployment. -Furthermore, if you select one of the Agents on your list, you can view its -[activity information](#view-the-agents-activity-information). +To install multiple agents, follow the +[advanced installation steps](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/build/deployment/gitlab-agent) +a second time and: -### View the Agent's activity information +1. Change the agent name and create a new configuration file. +1. Register the new agent. You receive a new token. Each token should be used only with one agent. +1. Change the namespace or prefix you use for the installation. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/277323) in GitLab 14.6. +You should also change the RBAC for the installed `agentk`. -The activity logs help you to identify problems and get the information -you need for troubleshooting. You can see events from a week before the -current date. To access an Agent's activity: - -1. In your Agent's repository, go to the Agents list as described [above](#view-your-agents). -1. Select the Agent you want to see the activity. - -The activity list includes: - -- Agent registration events: when a new token is **created**. -- Connection events: when an Agent is successfully **connected** to a cluster. - -Note that the connection status is logged when you connect an Agent for -the first time or after more than an hour of inactivity. - -To check what else is planned for the Agent's UI and provide feedback, -see the [related epic](https://gitlab.com/groups/gitlab-org/-/epics/4739). +## Example projects -### View vulnerabilities in cluster images **(ULTIMATE)** +The following example projects can help you get started with the agent. -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6346) in GitLab 14.8 [with a flag](../../../../administration/feature_flags.md) named `cluster_vulnerabilities`. Enabled by default. +- [Configuration repository with minimal manifests](https://gitlab.com/gitlab-examples/ops/gitops-demo/k8s-agents) +- [Distinct application and manifest repository example](https://gitlab.com/gitlab-examples/ops/gitops-demo/hello-world-service-gitops) +- [Auto DevOps setup that uses the CI/CD workflow](https://gitlab.com/gitlab-examples/ops/gitops-demo/hello-world-service) +- [Cluster management project template example that uses the CI/CD workflow](https://gitlab.com/gitlab-examples/ops/gitops-demo/cluster-management) -Users with at least the [Developer role](../../../permissions.md) -can view cluster vulnerabilities. You can access them through the [vulnerability report](../../../application_security/vulnerabilities/index.md) -or in your cluster's image through the following process: +## Upgrades and version compatibility -1. Configure [cluster image scanning](../../../application_security/cluster_image_scanning/index.md) - to your build process. -1. Go to your Agent's configuration repository. -1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. Select the **Agent** tab. -1. Select the Agent you want to see the vulnerabilities for. +The agent has two major components: `agentk` and `kas`. +GitLab provides `kas` installers built into the various GitLab installation methods. +The required `kas` version corresponds to the GitLab `major.minor` (X.Y) versions. - +At the same time, `agentk` and `kas` can differ by 1 minor version in either direction. For example, +`agentk` 14.4 supports `kas` 14.3, 14.4, and 14.5 (regardless of the patch). -## Create multiple Agents +A feature introduced in a given GitLab minor version might work with other `agentk` or `kas` versions. +To ensure it works, use at least the same `agentk` and `kas` minor version. For example, +if your GitLab version is 14.2, use at least `agentk` 14.2 and `kas` 14.2. -You can create and install multiple Agents using the same process -documented above. Give each Agent's configuration file a unique name -and you're good to go. You can create multiple Agents, for example: +We recommend upgrading your `kas` installations together with GitLab instances' upgrades, and to +[upgrade the `agentk` installations](#update-the-agent-version) after upgrading GitLab. -- To reach your cluster from different projects. -- To connect multiple clusters to GitLab. +The available `agentk` and `kas` versions are available in +[the Container Registry](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/container_registry/). -## Update the Agent version +### Update the agent version -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, GitLab warns you on the Agent's list page to update the Agent version installed on your cluster. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, GitLab warns you on the agent's list page to update the agent version installed on your cluster. -To update the Agent's version on your cluster, you need to re-run the [installation command](#install-the-agent-onto-the-cluster) +To update the agent's version, re-run the [installation command](#install-the-agent-in-the-cluster) with a newer `--agent-version`. Make sure to specify the other required parameters: `--kas-address`, `--namespace`, and `--agent-token`. -You can find the available `agentk` versions in [the container registry](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/container_registry/1223205?sort=desc). +The available `agentk` versions are in [the Container Registry](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/container_registry/1223205?sort=desc). -If you don't have access to your Agent's token, you can retrieve it from your cluster: +If you don't have access to your agent's token, you can retrieve it from your cluster: -1. On your computer, open the terminal and connect to your cluster. +1. Open a terminal and connect to your cluster. 1. To retrieve the namespace, run: ```shell @@ -251,28 +213,3 @@ If you don't have access to your Agent's token, you can retrieve it from your cl ```shell kubectl -n <namespace> get secret <secret-name> --template={{.data.token}} | base64 --decode ``` - -## Example projects - -The following example projects can help you get started with the Agent. - -- [Configuration repository](https://gitlab.com/gitlab-org/configure/examples/kubernetes-agent) -- This basic GitOps example deploys NGINX: [Manifest repository](https://gitlab.com/gitlab-org/configure/examples/gitops-project) - -## Upgrades and version compatibility - -The Agent is comprised of two major components: `agentk` and `kas`. -As we provide `kas` installers built into the various GitLab installation methods, the required `kas` version corresponds to the GitLab `major.minor` (X.Y) versions. - -At the same time, `agentk` and `kas` can differ by 1 minor version in either direction. For example, -`agentk` 14.4 supports `kas` 14.3, 14.4, and 14.5 (regardless of the patch). - -A feature introduced in a given GitLab minor version might work with other `agentk` or `kas` versions. -To make sure that it works, use at least the same `agentk` and `kas` minor version. For example, -if your GitLab version is 14.2, use at least `agentk` 14.2 and `kas` 14.2. - -We recommend upgrading your `kas` installations together with GitLab instances' upgrades, and to -[upgrade the `agentk` installations](#update-the-agent-version) after upgrading GitLab. - -The available `agentk` and `kas` versions can be found in -[the container registry](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/container_registry/). diff --git a/doc/user/clusters/agent/repository.md b/doc/user/clusters/agent/repository.md index a9d9fd1c13d..bf778cc0fdc 100644 --- a/doc/user/clusters/agent/repository.md +++ b/doc/user/clusters/agent/repository.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Agent configuration repository **(FREE)** +# Working with the agent for Kubernetes **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3834) in GitLab 13.11, the GitLab Agent became available on GitLab.com. @@ -12,324 +12,139 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added. -The [GitLab Agent](index.md) supports hosting your configuration for -multiple agents in a single repository. These agents can be running -in the same cluster or in multiple clusters, and potentially with more than one agent per cluster. +## View your agents -The Agent bootstraps with the GitLab installation URL and an authentication token, -and you provide the rest of the configuration in your repository, following -Infrastructure as Code (IaaC) best practices. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, the installed `agentk` version is displayed on the **Agent** tab. -A minimal repository layout looks like this, with `my-agent-1` as the name -of your Agent: +Prerequisite: -```plaintext -|- .gitlab - |- agents - |- my-agent-1 - |- config.yaml -``` +- You must have at least the Developer role. -Make sure that `<agent-name>` conforms to the [Agent's naming format](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name). +To view the list of agents: -## Synchronize manifest projects **(PREMIUM)** +1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select **Agent** tab to view clusters connected to GitLab through the agent. -Your `config.yaml` file contains a `gitops` section, which contains a `manifest_projects` -section. Each `id` in the `manifest_projects` section is the path to a Git repository -with Kubernetes resource definitions in YAML or JSON format. The Agent monitors -each project you declare, and when the project changes, GitLab deploys the changes -using the Agent. +On this page, you can view: -WARNING: -When using separate GitLab projects for manifest files and configuration repository, the manifests project must be public. +- All the registered agents for the current project. +- The connection status. +- The version of `agentk` installed on your cluster. +- The path to each agent configuration file. -To use multiple YAML files, specify a `paths` attribute in the `gitops.manifest_projects` section. +## View an agent's activity information -```yaml -gitops: - # Manifest projects are watched by the agent. Whenever a project changes, - # GitLab deploys the changes using the agent. - manifest_projects: - # No authentication mechanisms are currently supported. - # The `id` is a path to a Git repository with Kubernetes resource definitions - # in YAML or JSON format. - - id: gitlab-org/cluster-integration/gitlab-agent - # Namespace to use if not set explicitly in object manifest. - # Also used for inventory ConfigMap objects. - default_namespace: my-ns - # Paths inside of the repository to scan for manifest files. - # Directories with names starting with a dot are ignored. - paths: - # Read all .yaml files from team1/app1 directory. - # See https://github.com/bmatcuk/doublestar#about and - # https://pkg.go.dev/github.com/bmatcuk/doublestar/v2#Match for globbing rules. - - glob: '/team1/app1/*.yaml' - # Read all .yaml files from team2/apps and all subdirectories - - glob: '/team2/apps/**/*.yaml' - # If 'paths' is not specified or is an empty list, the configuration below is used - - glob: '/**/*.{yaml,yml,json}' - # Reconcile timeout defines whether the applier should wait - # until all applied resources have been reconciled, and if so, - # how long to wait. - reconcile_timeout: 3600s # 1 hour by default - # Dry run strategy defines whether changes should actually be performed, - # or if it is just talk and no action. - # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/common/common.go#L68-L89 - # Can be: none, client, server - dry_run_strategy: none # 'none' by default - # Prune defines whether pruning of previously applied - # objects should happen after apply. - prune: true # enabled by default - # Prune timeout defines whether we should wait for all resources - # to be fully deleted after pruning, and if so, how long we should - # wait. - prune_timeout: 3600s # 1 hour by default - # Prune propagation policy defines the deletion propagation policy - # that should be used for pruning. - # https://github.com/kubernetes/apimachinery/blob/44113beed5d39f1b261a12ec398a356e02358307/pkg/apis/meta/v1/types.go#L456-L470 - # Can be: orphan, background, foreground - prune_propagation_policy: foreground # 'foreground' by default - # Inventory policy defines if an inventory object can take over - # objects that belong to another inventory object or don't - # belong to any inventory object. - # This is done by determining if the apply/prune operation - # can go through for a resource based on the comparison - # the inventory-id value in the package and the owning-inventory - # annotation (config.k8s.io/owning-inventory) in the live object. - # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/inventory/policy.go#L12-L66 - # Can be: must_match, adopt_if_no_inventory, adopt_all - inventory_policy: must_match # 'must_match' by default -``` +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/277323) in GitLab 14.6. -### Using multiple manifest projects +The activity logs help you to identify problems and get the information +you need for troubleshooting. You can see events from a week before the +current date. To view an agent's activity: -Storing Kubernetes manifests in more than one repository can be handy, for example: +1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select the agent you want to see activity for. -- You may store manifests for different applications in separate repositories. -- Different teams can work on manifests of independent projects in separate repositories. +The activity list includes: -To use multiple repositories as the source of Kubernetes manifests, specify them in the list of -`manifest_projects` in your `config.yaml`: - -```yaml -gitops: - manifest_projects: - - id: group1/project1 - - id: group2/project2 -``` +- Agent registration events: When a new token is **created**. +- Connection events: When an agent is successfully **connected** to a cluster. -Note that repositories are synchronized **concurrently** and **independently** from each other, -which means that, ideally, there should **not** be any dependencies shared by these repositories. -Storing a logical group of manifests in a single repository may work better than distributing it across several -repositories. +The connection status is logged when you connect an agent for +the first time or after more than an hour of inactivity. -You cannot use a single repository as a source for multiple concurrent synchronization -operations. If such functionality is needed, you may use multiple agents reading -manifests from the same repository. +View and provide feedback about the UI in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/4739). -Ensure not to specify "overlapping" globs to avoid synchronizing the same files more than once. -This is detected by the Agent and leads to an error. +## Debug the agent -INCORRECT - both globs match `*.yaml` files in the root directory: +To debug the cluster-side component (`agentk`) of the agent, set the log +level according to the available options: -```yaml -gitops: - manifest_projects: - - id: project1 - paths: - - glob: '/**/*.yaml' - - glob: '/*.yaml' -``` +- `off` +- `warning` +- `error` +- `info` +- `debug` -CORRECT - single globs matches all `*.yaml` files recursively: +The log level defaults to `info`. You can change it by using a top-level `observability` +section in the configuration file, for example: ```yaml -gitops: - manifest_projects: - - id: project1 - paths: - - glob: '/**/*.yaml' +observability: + logging: + level: debug ``` -## Authorize projects and groups to use an Agent - -With the [CI/CD Tunnel](ci_cd_tunnel.md), you can authorize [projects](#authorize-projects-to-use-an-agent) -and [groups](#authorize-groups-to-use-an-agent) to use an Agent. +## Remove an agent -Then, you can reach your cluster from authorized projects and [run Kubernetes commands from GitLab CI/CD scripts](#run-kubectl-commands-using-the-cicd-tunnel) -in these projects. +You can remove an agent by using the [GitLab UI](#remove-an-agent-through-the-gitlab-ui) or the [GraphQL API](#remove-an-agent-with-the-gitlab-graphql-api). -### Authorize projects to use an Agent +### Remove an agent through the GitLab UI -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327850) in GitLab 14.4. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323055) in GitLab 14.7. -To grant projects access to the Agent through the CI/CD Tunnel: +To remove an agent from the UI: -1. Go to your Agent's configuration repository. -1. Edit the Agent's configuration file (`config.yaml`). -1. Add the `projects` attribute into `ci_access`. -1. Identify the project through its path: +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file. +1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. In the table, in the row for your agent, in the **Options** column, select the vertical ellipsis (**{ellipsis_v}**). +1. Select **Delete agent**. - ```yaml - ci_access: - projects: - - id: path/to/project - ``` - -### Authorize groups to use an Agent - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. - -To grant access to all projects within a group: +### Remove an agent with the GitLab GraphQL API -1. Go to your Agent's configuration repository. -1. Edit the Agent's configuration file (`config.yaml`). -1. Add the `groups` attribute into `ci_access`. -1. Identify the group or subgroup through its path: +1. Get the `<cluster-agent-token-id>` from a query in the interactive GraphQL explorer. + - For GitLab.com, go to <https://gitlab.com/-/graphql-explorer> to open GraphQL Explorer. + - For self-managed GitLab, go to `https://gitlab.example.com/-/graphql-explorer`, replacing `gitlab.example.com` with your instance's URL. - ```yaml - ci_access: - groups: - - id: path/to/group/subgroup + ```graphql + query{ + project(fullPath: "<full-path-to-agent-configuration-project>") { + clusterAgent(name: "<agent-name>") { + id + tokens { + edges { + node { + id + } + } + } + } + } + } ``` -## Run `kubectl` commands using the CI/CD Tunnel - -After you authorize your project or group to use the Agent, you need to -configure the project's `.gitlab-ci.yaml` file to access the Agent. -This makes it possible to deploy applications to your cluster and run -any Kubernetes-specific commands from the authorized project. - -First, configure your Agent: - -1. Go to your Agent's configuration repository. -1. Edit your Agent's `config.yaml` file authorizing the [project](#authorize-projects-to-use-an-agent) or [group](#authorize-groups-to-use-an-agent) you want to run Kubernetes commands from. - -Then, configure the other project: - -1. Go to the project where you want to run Kubernetes commands from. -1. Edit your project's `.gitlab-ci.yml` file. -1. Set your Agent's context in the first command of the script with the format `path/to/agent/repository:agent-name`. -1. Run Kubernetes commands. - -For example: - -```yaml - deploy: - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config use-context path/to/agent/repository:agent-name - - kubectl get pods -``` - -When you use the Agent, KubeContexts are named as `path/to/agent/repository:agent-name`. - -To get the list of available contexts: - -1. Open your terminal and connect to your cluster. -1. Run `kubectl config get-contexts`. - -### `kubectl` commands not supported - -The commands `kubectl exec`, `kubectl cp`, and `kubectl attach` are not supported by the CI/CD tunnel. -Anything else that uses the same API endpoints does not work either as they use the deprecated -SPDY protocol. -We [plan to add support for these features](https://gitlab.com/gitlab-org/gitlab/-/issues/346248) -in a future version of GitLab. - -### `kubectl` requires TLS - -`kubectl` would never send credentials over an unencrypted connection. Self-managed users should ensure that their -GitLab instance is configured with TLS for the CI/CD tunnel feature to work. Trying to use it without TLS -would produce errors: - -```shell -$ kubectl get pods -error: You must be logged in to the server (the server has asked for the client to provide credentials) -``` - -## Use impersonation to restrict project and group access **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345014) in GitLab 14.5. - -By default, the [CI/CD Tunnel](ci_cd_tunnel.md) inherits all the permissions from the service account used to install the -Agent in the cluster. -To restrict access to your cluster, you can use [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). - -To specify impersonations, use the `access_as` attribute in your Agent's configuration file and use Kubernetes RBAC rules to manage impersonated account permissions. - -You can impersonate: - -- The Agent itself (default). -- The CI job that accesses the cluster. -- A specific user or system account defined within the cluster. - -### Impersonate the Agent - -The Agent is impersonated by default. You don't need to do anything to impersonate it. - -### Impersonate the CI job that accesses the cluster - -To impersonate the CI job that accesses the cluster, add the `ci_job: {}` key-value -under the `access_as` key. - -When the agent makes the request to the actual Kubernetes API, it sets the -impersonation credentials in the following way: - -- `UserName` is set to `gitlab:ci_job:<job id>`. Example: `gitlab:ci_job:1074499489`. -- `Groups` is set to: - - `gitlab:ci_job` to identify all requests coming from CI jobs. - - The list of IDs of groups the project is in. - - The project ID. - - The slug of the environment this job belongs to. - - Example: for a CI job in `group1/group1-1/project1` where: +1. Remove an agent record with GraphQL by deleting the `clusterAgentToken`. - - Group `group1` has ID 23. - - Group `group1/group1-1` has ID 25. - - Project `group1/group1-1/project1` has ID 150. - - Job running in a prod environment. + ```graphql + mutation deleteAgent { + clusterAgentDelete(input: { id: "<cluster-agent-id>" } ) { + errors + } + } - Group list would be `[gitlab:ci_job, gitlab:group:23, gitlab:group:25, gitlab:project:150, gitlab:project_env:150:prod]`. - -- `Extra` carries extra information about the request. The following properties are set on the impersonated identity: - -| Property | Description | -| -------- | ----------- | -| `agent.gitlab.com/id` | Contains the agent ID. | -| `agent.gitlab.com/config_project_id` | Contains the agent's configuration project ID. | -| `agent.gitlab.com/project_id` | Contains the CI project ID. | -| `agent.gitlab.com/ci_pipeline_id` | Contains the CI pipeline ID. | -| `agent.gitlab.com/ci_job_id` | Contains the CI job ID. | -| `agent.gitlab.com/username` | Contains the username of the user the CI job is running as. | -| `agent.gitlab.com/environment_slug` | Contains the slug of the environment. Only set if running in an environment. | - -Example to restrict access by the CI job's identity: - -```yaml -ci_access: - projects: - - id: path/to/project - access_as: - ci_job: {} -``` - -### Impersonate a static identity - -For the given CI/CD Tunnel connection, you can use a static identity for the impersonation. + mutation deleteToken { + clusterAgentTokenDelete(input: { id: "<cluster-agent-token-id>" }) { + errors + } + } + ``` -Add the `impersonate` key under the `access_as` key to make the request using the provided identity. +1. Verify whether the removal occurred successfully. If the output in the Pod logs includes `unauthenticated`, it means that the agent was successfully removed: -The identity can be specified with the following keys: + ```json + { + "level": "warn", + "time": "2021-04-29T23:44:07.598Z", + "msg": "GetConfiguration.Recv failed", + "error": "rpc error: code = Unauthenticated desc = unauthenticated" + } + ``` -- `username` (required) -- `uid` -- `groups` -- `extra` +1. Delete the agent in your cluster: -See the [official Kubernetes documentation for more details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) on the usage of these keys. + ```shell + kubectl delete -n gitlab-kubernetes-agent -f ./resources.yml + ``` ## Surface network security alerts from cluster to GitLab **(ULTIMATE)** @@ -340,7 +155,28 @@ Cilium integration is in its end-of-life process. It's [deprecated](https://gitl for use in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. -The GitLab Agent provides an [integration with Cilium](index.md#kubernetes-network-security-alerts). +The agent for Kubernetes also provides an integration with Cilium. This integration provides a simple way to +generate network policy-related alerts and to surface those alerts in GitLab. + +Several components work in concert for the agent to generate the alerts: + +- A working Kubernetes cluster. +- Cilium integration through either of these options: + - Installation through [cluster management template](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). + - Enablement of [hubble-relay](https://docs.cilium.io/en/v1.8/concepts/overview/#hubble) on an + existing installation. +- One or more network policies through any of these options: + - Use the [Container Network Policy editor](../../application_security/policies/index.md#container-network-policy-editor) to create and manage policies. + - Use an [AutoDevOps](../../application_security/policies/index.md#container-network-policy) configuration. + - Add the required labels and annotations to existing network policies. +- A configuration repository with [Cilium configured in `config.yaml`](repository.md#surface-network-security-alerts-from-cluster-to-gitlab) + +The setup process follows the same [agent's installation steps](install/index.md), +with the following differences: + +- When you define a configuration repository, you must do so with [Cilium settings](repository.md#surface-network-security-alerts-from-cluster-to-gitlab). +- You do not need to specify the `gitops` configuration section. + To integrate, add a top-level `cilium` section to your `config.yml` file. Currently, the only configuration option is the Hubble relay address: @@ -357,107 +193,3 @@ you can use `hubble-relay.gitlab-managed-apps.svc.cluster.local:80` as the addre cilium: hubble_relay_address: "hubble-relay.gitlab-managed-apps.svc.cluster.local:80" ``` - -## Scan your container images for vulnerabilities **(ULTIMATE)** - -You can use [cluster image scanning](../../application_security/cluster_image_scanning/index.md) -to scan container images in your cluster for security vulnerabilities. - -To begin scanning all resources in your cluster, add a `starboard` -configuration block to your agent's `config.yaml` with no `filters`: - -```yaml -starboard: - vulnerability_report: - filters: [] -``` - -The namespaces that are able to be scanned depend on the [Starboard Operator install mode](https://aquasecurity.github.io/starboard/latest/operator/configuration/#install-modes). -By default, the Starboard Operator only scans resources in the `default` namespace. To change this -behavior, edit the `STARBOARD_OPERATOR` environment variable in the `starboard-operator` deployment -definition. - -By adding filters, you can limit scans by: - -- Resource name -- Kind -- Container name -- Namespace - -```yaml -starboard: - vulnerability_report: - filters: - - namespaces: - - staging - - production - kinds: - - Deployment - - DaemonSet - containers: - - ruby - - postgres - - nginx - resources: - - my-app-name - - postgres - - ingress-nginx -``` - -A resource is scanned if the resource matches any of the given names and all of the given filter -types (`namespaces`, `kinds`, `containers`, `resources`). If a filter type is omitted, then all -names are scanned. In this example, a resource isn't scanned unless it has a container named `ruby`, -`postgres`, or `nginx`, and it's a `Deployment`: - -```yaml -starboard: - vulnerability_report: - filters: - - kinds: - - Deployment - containers: - - ruby - - postgres - - nginx -``` - -There is also a global `namespaces` field that applies to all filters: - -```yaml -starboard: - vulnerability_report: - namespaces: - - production - filters: - - kinds: - - Deployment - - kinds: - - DaemonSet - resources: - - log-collector -``` - -In this example, the following resources are scanned: - -- All deployments (`Deployment`) in the `production` namespace -- All daemon sets (`DaemonSet`) named `log-collector` in the `production` namespace - -## Debugging - -To debug the cluster-side component (`agentk`) of the Agent, set the log -level according to the available options: - -- `off` -- `warning` -- `error` -- `info` -- `debug` - -The log level defaults to `info`. You can change it by using a top-level `observability` -section in the configuration file, for example: - -```yaml -observability: - logging: - level: debug -``` diff --git a/doc/user/clusters/agent/troubleshooting.md b/doc/user/clusters/agent/troubleshooting.md index 2c9f98b7c45..2fcf0c9aa98 100644 --- a/doc/user/clusters/agent/troubleshooting.md +++ b/doc/user/clusters/agent/troubleshooting.md @@ -188,6 +188,5 @@ Alternatively, you can mount the certificate file at a different location and in } ``` -This error is shown if the manifest project is not public. To fix it, -[make sure your manifest project is public](repository.md#synchronize-manifest-projects) or your manifest files +This error is shown if the manifest project is not public. To fix it, make sure your manifest project is public or your manifest files are stored in the Agent's configuration repository. diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md new file mode 100644 index 00000000000..ef3fd800220 --- /dev/null +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -0,0 +1,113 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Container vulnerability scanning **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6346) in GitLab 14.8 [with a flag](../../../administration/feature_flags.md) named `cluster_vulnerabilities`. Enabled by default. + +To view cluster vulnerabilities, you can view the [vulnerability report](../../application_security/vulnerabilities/index.md). +You can also configure your agent so the vulnerabilities are displayed with other agent information in GitLab. + +## View cluster vulnerabilities + +Prerequisite: + +- You must have at least the Developer role. +- [Cluster image scanning](../../application_security/cluster_image_scanning/index.md) + must be part of your build process. + +To view vulnerability information in GitLab: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file. +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select the **Agent** tab. +1. Select the agent you want to see the vulnerabilities for. + + + +## Enable cluster vulnerability scanning **(ULTIMATE)** + +You can use [cluster image scanning](../../application_security/cluster_image_scanning/index.md) +to scan container images in your cluster for security vulnerabilities. + +To begin scanning all resources in your cluster, add a `starboard` +configuration block to your agent configuration file with no `filters`: + +```yaml +starboard: + vulnerability_report: + filters: [] +``` + +The namespaces that are able to be scanned depend on the [Starboard Operator install mode](https://aquasecurity.github.io/starboard/latest/operator/configuration/#install-modes). +By default, the Starboard Operator only scans resources in the `default` namespace. To change this +behavior, edit the `STARBOARD_OPERATOR` environment variable in the `starboard-operator` deployment +definition. + +By adding filters, you can limit scans by: + +- Resource name +- Kind +- Container name +- Namespace + +```yaml +starboard: + vulnerability_report: + filters: + - namespaces: + - staging + - production + kinds: + - Deployment + - DaemonSet + containers: + - ruby + - postgres + - nginx + resources: + - my-app-name + - postgres + - ingress-nginx +``` + +A resource is scanned if the resource matches any of the given names and all of the given filter +types (`namespaces`, `kinds`, `containers`, `resources`). If a filter type is omitted, then all +names are scanned. In this example, a resource isn't scanned unless it has a container named `ruby`, +`postgres`, or `nginx`, and it's a `Deployment`: + +```yaml +starboard: + vulnerability_report: + filters: + - kinds: + - Deployment + containers: + - ruby + - postgres + - nginx +``` + +There is also a global `namespaces` field that applies to all filters: + +```yaml +starboard: + vulnerability_report: + namespaces: + - production + filters: + - kinds: + - Deployment + - kinds: + - DaemonSet + resources: + - log-collector +``` + +In this example, the following resources are scanned: + +- All deployments (`Deployment`) in the `production` namespace. +- All daemon sets (`DaemonSet`) named `log-collector` in the `production` namespace. diff --git a/doc/user/clusters/management_project_template.md b/doc/user/clusters/management_project_template.md index 2d7e89ef765..af82c166dcb 100644 --- a/doc/user/clusters/management_project_template.md +++ b/doc/user/clusters/management_project_template.md @@ -69,7 +69,7 @@ We assume that you already have a cluster connected through the Agent and 1. [Create a new project from the Cluster Management Project Template](#create-a-new-project-based-on-the-cluster-management-template). This new project is "project B". -1. In your "project A", [grant the Agent access to the new project (B) through the CI/CD Tunnel](agent/repository.md#authorize-projects-to-use-an-agent). +1. In your "project A", [grant the Agent access to the new project (B) through the CI/CD Tunnel](agent/ci_cd_tunnel.md#authorize-the-agent). 1. From the "project's B" settings, add a [new environment variable](../../ci/variables/index.md#add-a-cicd-variable-to-a-project) `$KUBE_CONTEXT` and set it to `path/to/agent-configuration-project:your-agent-name`. 1. In "project B", [configure the components](#configure-the-available-components) inherited from the template. diff --git a/doc/user/group/planning_hierarchy/img/view-project-work-item-hierarchy_v14_8.png b/doc/user/group/planning_hierarchy/img/view-project-work-item-hierarchy_v14_8.png Binary files differdeleted file mode 100644 index 2ddd551ee46..00000000000 --- a/doc/user/group/planning_hierarchy/img/view-project-work-item-hierarchy_v14_8.png +++ /dev/null diff --git a/doc/user/group/planning_hierarchy/index.md b/doc/user/group/planning_hierarchy/index.md index 934421e8a9a..6ffc47923f2 100644 --- a/doc/user/group/planning_hierarchy/index.md +++ b/doc/user/group/planning_hierarchy/index.md @@ -5,7 +5,7 @@ group: Product Planning info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Planning hierarchies **(FREE)** +# Planning hierarchies **(PREMIUM)** Planning hierarchies are an integral part of breaking down your work in GitLab. To understand how you can use epics and issues together in hierarchies, remember the following: @@ -20,21 +20,7 @@ To learn about hierarchies in general, common frameworks, and using GitLab for portfolio management, see [How to use GitLab for Agile portfolio planning and project management](https://about.gitlab.com/blog/2020/11/11/gitlab-for-agile-portfolio-planning-project-management/). -## View planning hierarchies - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340844/) in GitLab 14.8. - -To view the planning hierarchy in a project: - -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Project information > Planning hierarchy**. - -Under **Current structure**, you can see a hierarchy diagram that matches your current planning hierarchy. -The work items outside your subscription plan show up below **Unavailable structure**. - - - -## Hierarchies with epics **(PREMIUM)** +## Hierarchies with epics With epics, you can achieve the following hierarchy: @@ -74,7 +60,7 @@ In an issue, you can view the parented epic above the issue in the right sidebar  -## View ancestry of an epic **(PREMIUM)** +## View ancestry of an epic In an epic, you can view the ancestors as parents in the right sidebar under **Ancestors**. diff --git a/doc/user/infrastructure/clusters/index.md b/doc/user/infrastructure/clusters/index.md index 144a8cd2d31..06122b85c98 100644 --- a/doc/user/infrastructure/clusters/index.md +++ b/doc/user/infrastructure/clusters/index.md @@ -68,5 +68,5 @@ The concept of [project-level](../../project/clusters/index.md), extinct in the new model, although the functionality remains to some extent. The Agent is always configured in a single GitLab project, but you can use the CI/CD Tunnel to -[authorize other projects and groups to use the same Agent](../../clusters/agent/repository.md#authorize-projects-and-groups-to-use-an-agent). +authorize other projects and groups to use the same Agent. By doing so, you are granting these projects and groups access to the same cluster, which is similar to group-level clusters' use case. diff --git a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md index 1dd1c760bcc..881a96fc986 100644 --- a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md +++ b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md @@ -34,7 +34,7 @@ all the features available to clusters connected through certificates. With GitLab-managed clusters, GitLab creates separate service accounts and namespaces for every branch and deploys using these resources. -To achieve a similar result with the GitLab Agent, you can use [impersonation](../../clusters/agent/repository.md#use-impersonation-to-restrict-project-and-group-access) +To achieve a similar result with the GitLab Agent, you can use [impersonation](../../clusters/agent/ci_cd_tunnel.md#use-impersonation-to-restrict-project-and-group-access) strategies to deploy to your cluster with restricted account access. To do so: 1. Choose the impersonation strategy that suits your needs. diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 6e5196ecdbd..1031f38792b 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -80,6 +80,10 @@ module Gitlab @all ||= definitions.map { |_key_path, definition| definition } end + def not_removed + all.select { |definition| definition.attributes[:status] != 'removed' }.index_by(&:key_path) + end + def with_instrumentation_class all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } end diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb new file mode 100644 index 00000000000..e04e2e589b2 --- /dev/null +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Service Ping payload build using the instrumentation classes +# for given metrics key_paths and output method +module Gitlab + module Usage + module ServicePing + class InstrumentedPayload + attr_reader :metrics_key_paths + attr_reader :output_method + + def initialize(metrics_key_paths, output_method) + @metrics_key_paths = metrics_key_paths + @output_method = output_method + end + + def build + metrics_key_paths.map do |key_path| + compute_instrumental_value(key_path, output_method) + end.reduce({}, :deep_merge) + end + + private + + # Not all metrics defintions have instrumentation classes + # The value can be computed only for those that have it + def instrumented_metrics_defintions + Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + + def compute_instrumental_value(key_path, output_method) + definition = instrumented_metrics_defintions.find { |df| df.key_path == key_path } + + return {} unless definition.present? + + Gitlab::Usage::Metric.new(definition).method(output_method).call + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb new file mode 100644 index 00000000000..b00aee45e59 --- /dev/null +++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Process the UsageData payload to get the keys that have a metric defintion +# Get the missing keys from the payload +module Gitlab + module Usage + module ServicePing + class PayloadKeysProcessor + attr_reader :old_payload + + def initialize(old_payload) + @old_payload = old_payload + end + + def key_paths + @key_paths ||= payload_keys.to_a.flatten.compact + end + + def missing_instrumented_metrics_key_paths + @missing_key_paths ||= metrics_with_instrumentation.map(&:key_path) - key_paths + end + + private + + def payload_keys(payload = old_payload, parents = []) + return unless payload.is_a?(Hash) + + payload.map do |key, value| + if has_metric_definition?(key, parents) + parents.dup.append(key).join('.') + else + payload_keys(value, parents.dup << key) if value.is_a?(Hash) + end + end + end + + def has_metric_definition?(key, parent_keys) + key_path = parent_keys.dup.append(key).join('.') + metric_definitions.key?(key_path) + end + + def metric_definitions + ::Gitlab::Usage::MetricDefinition.not_removed + end + + def metrics_with_instrumentation + ::Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index d9e30c46498..794f3373043 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -7,16 +7,29 @@ module Gitlab def for(output:, cached: false) case output.to_sym when :all_metrics_values - all_metrics_values(cached) + with_instrumentation_classes(all_metrics_values(cached), :with_value) when :metrics_queries - metrics_queries + with_instrumentation_classes(metrics_queries, :with_instrumentation) when :non_sql_metrics_values - non_sql_metrics_values + with_instrumentation_classes(non_sql_metrics_values, :with_instrumentation) end end private + def with_instrumentation_classes(old_payload, output_method) + if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + + instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths + + instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build + + old_payload.deep_merge(instrumented_payload) + else + old_payload + end + end + def all_metrics_values(cached) Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e66a565246b..1250d3fd562 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -70,10 +70,9 @@ module Gitlab def system_usage_data issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - { + counts = { counts: { assignee_lists: count(List.assignee), - boards: add_metric('CountBoardsMetric', time_frame: 'all'), ci_builds: count(::Ci::Build), ci_internal_pipelines: count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), @@ -167,6 +166,12 @@ module Gitlab data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } + + if Feature.disabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + counts[:counts][:boards] = add_metric('CountBoardsMetric', time_frame: 'all') + end + + counts end # rubocop: enable Metrics/AbcSize diff --git a/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb b/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb index 57a83da3425..61445603a2d 100644 --- a/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb +++ b/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb @@ -170,6 +170,14 @@ RSpec.describe Groups::DependencyProxyForContainersController do end end + shared_examples 'namespace statistics refresh' do + it 'updates namespace statistics' do + expect(Groups::UpdateStatisticsWorker).to receive(:perform_async) + + subject + end + end + before do allow(Gitlab.config.dependency_proxy) .to receive(:enabled).and_return(true) @@ -403,13 +411,15 @@ RSpec.describe Groups::DependencyProxyForContainersController do context 'with a valid user' do before do group.add_guest(user) - - expect_next_found_instance_of(Group) do |instance| - expect(instance).to receive_message_chain(:dependency_proxy_blobs, :create!) - end end it_behaves_like 'a package tracking event', described_class.name, 'pull_blob' + + it 'creates a blob' do + expect { subject }.to change { group.dependency_proxy_blobs.count }.by(1) + end + + it_behaves_like 'namespace statistics refresh' end end @@ -473,6 +483,8 @@ RSpec.describe Groups::DependencyProxyForContainersController do expect(manifest.digest).to eq(digest) expect(manifest.file_name).to eq(file_name) end + + it_behaves_like 'namespace statistics refresh' end context 'with existing stale manifest' do @@ -483,6 +495,8 @@ RSpec.describe Groups::DependencyProxyForContainersController do expect { subject }.to change { group.dependency_proxy_manifests.count }.by(0) .and change { manifest.reload.digest }.from(old_digest).to(digest) end + + it_behaves_like 'namespace statistics refresh' end end end diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb index a22b3a733bd..1127d1cd477 100644 --- a/spec/lib/gitlab/usage/metric_definition_spec.rb +++ b/spec/lib/gitlab/usage/metric_definition_spec.rb @@ -50,6 +50,28 @@ RSpec.describe Gitlab::Usage::MetricDefinition do expect { described_class.definitions }.not_to raise_error end + describe 'not_removed' do + let(:all_definitions) do + metrics_definitions = [ + { key_path: 'metric1', instrumentation_class: 'RedisHLLMetric', status: 'active' }, + { key_path: 'metric2', instrumentation_class: 'RedisHLLMetric', status: 'broken' }, + { key_path: 'metric3', instrumentation_class: 'RedisHLLMetric', status: 'active' }, + { key_path: 'metric4', instrumentation_class: 'RedisHLLMetric', status: 'removed' } + ] + metrics_definitions.map { |definition| described_class.new(definition[:key_path], definition.symbolize_keys) } + end + + before do + allow(described_class).to receive(:all).and_return(all_definitions) + end + + it 'includes metrics that are not removed' do + expect(described_class.not_removed.count).to eq(3) + + expect(described_class.not_removed.keys).to match_array(%w(metric1 metric2 metric3)) + end + end + describe '#with_instrumentation_class' do let(:metric_status) { 'active' } let(:all_definitions) do diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb index 0f95da74ff9..f81ad9b193d 100644 --- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb +++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb @@ -27,8 +27,8 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do context 'for count with default column metrics' do it_behaves_like 'name suggestion' do # corresponding metric is collected with count(Board) - let(:key_path) { 'counts.boards' } - let(:name_suggestion) { /count_boards/ } + let(:key_path) { 'counts.issues' } + let(:name_suggestion) { /count_issues/ } end end diff --git a/spec/lib/gitlab/usage/service_ping/instrumented_payload_spec.rb b/spec/lib/gitlab/usage/service_ping/instrumented_payload_spec.rb new file mode 100644 index 00000000000..76548483cfa --- /dev/null +++ b/spec/lib/gitlab/usage/service_ping/instrumented_payload_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::ServicePing::InstrumentedPayload do + let(:uuid) { "0000-0000-0000" } + + before do + allow(ApplicationRecord.connection).to receive(:transaction_open?).and_return(false) + allow(Gitlab::CurrentSettings).to receive(:uuid).and_return(uuid) + end + + context 'when building service ping with values' do + let(:metrics_key_paths) { %w(counts.boards uuid redis_hll_counters.search.i_search_total_monthly) } + let(:expected_payload) do + { + counts: { boards: 0 }, + redis_hll_counters: { search: { i_search_total_monthly: 0 } }, + uuid: uuid + } + end + + it 'builds the service ping payload for the metrics key_paths' do + expect(described_class.new(metrics_key_paths, :with_value).build).to eq(expected_payload) + end + end + + context 'when building service ping with instrumentations' do + let(:metrics_key_paths) { %w(counts.boards uuid redis_hll_counters.search.i_search_total_monthly) } + let(:expected_payload) do + { + counts: { boards: "SELECT COUNT(\"boards\".\"id\") FROM \"boards\"" }, + redis_hll_counters: { search: { i_search_total_monthly: 0 } }, + uuid: uuid + } + end + + it 'builds the service ping payload for the metrics key_paths' do + expect(described_class.new(metrics_key_paths, :with_instrumentation).build).to eq(expected_payload) + end + end + + context 'when missing instrumentation class' do + it 'returns empty hash' do + expect(described_class.new(['counts.ci_builds'], :with_instrumentation).build).to eq({}) + expect(described_class.new(['counts.ci_builds'], :with_value).build).to eq({}) + end + end +end diff --git a/spec/lib/gitlab/usage/service_ping/payload_keys_processor_spec.rb b/spec/lib/gitlab/usage/service_ping/payload_keys_processor_spec.rb new file mode 100644 index 00000000000..2dc95294d03 --- /dev/null +++ b/spec/lib/gitlab/usage/service_ping/payload_keys_processor_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::ServicePing::PayloadKeysProcessor do + context 'with an object metric' do + let(:payload) { { counts: { issues: 1, boards: 1 }, topology: { duration_d: 100 }, redis_hll_counters: { search: { i_search_total_monthly: 1 } } } } + + it 'returns the payload keys that have a metric definition' do + expect(described_class.new(payload).key_paths).to match_array(['counts.issues', 'counts.boards', 'topology', 'redis_hll_counters.search.i_search_total_monthly']) + end + end + + context 'with a missing metric definition' do + let(:payload) { { counts: { issues: 1, boards: 1 }, missing_definition: 1, topology: { duration_d: 100 } } } + + it 'returns the payload keys that have a metric definition' do + expect(described_class.new(payload).key_paths).to match_array(['counts.issues', 'counts.boards', 'topology']) + end + end + + context 'with array metric' do + let(:payload) { { counts: { issues: 1, boards: 1 }, settings: { collected_data_categories: ['standard'] }, topology: { duration_d: 100 } } } + + it 'returns the payload keys that have a metric definition' do + expect(described_class.new(payload).key_paths).to match_array(['counts.issues', 'counts.boards', 'topology', 'settings.collected_data_categories']) + end + end + + context 'missing_instrumented_metrics_key_paths' do + let(:payload) do + { + counts: { issues: 1, boards: 1 }, + topology: { duration_d: 100 }, + redis_hll_counters: { search: { i_search_total_monthly: 1 } } + } + end + + let(:metrics_definitions) do + [ + double(:issues, key_path: 'counts.issues'), + double(:topology, key_path: 'topology'), + double(:i_search_total_monthly, key_path: 'redis_hll_counters.search.i_search_total_monthly'), + double(:collected_data_categories, key_path: 'settings.collected_data_categories') + ] + end + + before do + allow(::Gitlab::Usage::MetricDefinition).to receive(:with_instrumentation_class).and_return(metrics_definitions) + end + + it 'returns the missing keys' do + expect(described_class.new(payload).missing_instrumented_metrics_key_paths).to match_array(['settings.collected_data_categories']) + end + end +end diff --git a/spec/lib/gitlab/usage/service_ping_report_spec.rb b/spec/lib/gitlab/usage/service_ping_report_spec.rb index 9b9b24ad128..ea0ac13e51a 100644 --- a/spec/lib/gitlab/usage/service_ping_report_spec.rb +++ b/spec/lib/gitlab/usage/service_ping_report_spec.rb @@ -3,66 +3,121 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_caching do - let(:usage_data) { { uuid: "1111" } } + let(:usage_data) { { uuid: "1111", counts: { issue: 0 } } } - context 'for output: :all_metrics_values' do - it 'generates the service ping' do - expect(Gitlab::UsageData).to receive(:data) + context 'when feature merge_service_ping_instrumented_metrics enabled' do + before do + stub_feature_flags(merge_service_ping_instrumented_metrics: true) - described_class.for(output: :all_metrics_values) + allow_next_instance_of(Gitlab::Usage::ServicePing::PayloadKeysProcessor) do |instance| + allow(instance).to receive(:missing_key_paths).and_return([]) + end + + allow_next_instance_of(Gitlab::Usage::ServicePing::InstrumentedPayload) do |instance| + allow(instance).to receive(:build).and_return({}) + end end - end - context 'for output: :metrics_queries' do - it 'generates the service ping' do - expect(Gitlab::UsageDataQueries).to receive(:data) + context 'all_metrics_values' do + it 'generates the service ping when there are no missing values' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq({ uuid: "1111", counts: { issue: 0 } }) + end + + it 'generates the service ping with the missing values' do + expect_next_instance_of(Gitlab::Usage::ServicePing::PayloadKeysProcessor, usage_data) do |instance| + expect(instance).to receive(:missing_instrumented_metrics_key_paths).and_return(['counts.boards']) + end + + expect_next_instance_of(Gitlab::Usage::ServicePing::InstrumentedPayload, ['counts.boards'], :with_value) do |instance| + expect(instance).to receive(:build).and_return({ counts: { boards: 1 } }) + end - described_class.for(output: :metrics_queries) + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq({ uuid: "1111", counts: { issue: 0, boards: 1 } }) + end end - end - context 'for output: :non_sql_metrics_values' do - it 'generates the service ping' do - expect(Gitlab::UsageDataNonSqlMetrics).to receive(:data) + context 'for output: :metrics_queries' do + it 'generates the service ping' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) - described_class.for(output: :non_sql_metrics_values) + described_class.for(output: :metrics_queries) + end end - end - context 'when using cached' do - context 'for cached: true' do - let(:new_usage_data) { { uuid: "1112" } } + context 'for output: :non_sql_metrics_values' do + it 'generates the service ping' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) + + described_class.for(output: :non_sql_metrics_values) + end + end - it 'caches the values' do - allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) + context 'when using cached' do + context 'for cached: true' do + let(:new_usage_data) { { uuid: "1112" } } - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data) + it 'caches the values' do + allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(Rails.cache.fetch('usage_data')).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data) + + expect(Rails.cache.fetch('usage_data')).to eq(usage_data) + end + + it 'writes to cache and returns fresh data' do + allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) + + expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data) + + expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + end end - it 'writes to cache and returns fresh data' do - allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) + context 'when no caching' do + let(:new_usage_data) { { uuid: "1112" } } - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) - expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data) + it 'returns fresh data' do + allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) + + expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + end end end + end - context 'when no caching' do - let(:new_usage_data) { { uuid: "1112" } } + context 'when feature merge_service_ping_instrumented_metrics disabled' do + before do + stub_feature_flags(merge_service_ping_instrumented_metrics: false) + end - it 'returns fresh data' do - allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data) + context 'all_metrics_values' do + it 'generates the service ping when there are no missing values' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) + expect(described_class.for(output: :all_metrics_values)).to eq({ uuid: "1111", counts: { issue: 0 } }) + end + end + + context 'for output: :metrics_queries' do + it 'generates the service ping' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) + + described_class.for(output: :metrics_queries) + end + end - expect(described_class.for(output: :all_metrics_values)).to eq(usage_data) - expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data) + context 'for output: :non_sql_metrics_values' do + it 'generates the service ping' do + expect(Gitlab::UsageData).to receive(:data).and_return(usage_data) - expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data) + described_class.for(output: :non_sql_metrics_values) end end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index bea07dd9c43..868c667e69d 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -507,6 +507,8 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do end it 'gathers usage counts', :aggregate_failures do + stub_feature_flags(merge_service_ping_instrumented_metrics: false) + count_data = subject[:counts] expect(count_data[:boards]).to eq(1) expect(count_data[:projects]).to eq(4) diff --git a/spec/models/dependency_proxy/blob_spec.rb b/spec/models/dependency_proxy/blob_spec.rb index 10d06406ad7..cc62aecd1ab 100644 --- a/spec/models/dependency_proxy/blob_spec.rb +++ b/spec/models/dependency_proxy/blob_spec.rb @@ -5,6 +5,10 @@ RSpec.describe DependencyProxy::Blob, type: :model do it_behaves_like 'ttl_expirable' it_behaves_like 'destructible', factory: :dependency_proxy_blob + it_behaves_like 'updates namespace statistics' do + let(:statistic_source) { build(:dependency_proxy_blob, size: 10) } + end + describe 'relationships' do it { is_expected.to belong_to(:group) } end diff --git a/spec/models/dependency_proxy/manifest_spec.rb b/spec/models/dependency_proxy/manifest_spec.rb index ab7881b1d39..d43079f607a 100644 --- a/spec/models/dependency_proxy/manifest_spec.rb +++ b/spec/models/dependency_proxy/manifest_spec.rb @@ -5,6 +5,10 @@ RSpec.describe DependencyProxy::Manifest, type: :model do it_behaves_like 'ttl_expirable' it_behaves_like 'destructible', factory: :dependency_proxy_manifest + it_behaves_like 'updates namespace statistics' do + let(:statistic_source) { build(:dependency_proxy_manifest, size: 10) } + end + describe 'relationships' do it { is_expected.to belong_to(:group) } end diff --git a/spec/services/service_ping/build_payload_service_spec.rb b/spec/services/service_ping/build_payload_service_spec.rb index cd2685069c9..b90e5e66518 100644 --- a/spec/services/service_ping/build_payload_service_spec.rb +++ b/spec/services/service_ping/build_payload_service_spec.rb @@ -4,6 +4,10 @@ require 'spec_helper' RSpec.describe ServicePing::BuildPayloadService do describe '#execute', :without_license do + before do + stub_feature_flags(merge_service_ping_instrumented_metrics: false) + end + subject(:service_ping_payload) { described_class.new.execute } include_context 'stubbed service ping metrics definitions' do diff --git a/spec/support/shared_examples/models/concerns/update_namespace_statistics_shared_examples.rb b/spec/support/shared_examples/models/concerns/update_namespace_statistics_shared_examples.rb new file mode 100644 index 00000000000..255b6efa518 --- /dev/null +++ b/spec/support/shared_examples/models/concerns/update_namespace_statistics_shared_examples.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'updates namespace statistics' do + let(:namespace_statistics_name) { described_class.namespace_statistics_name } + let(:statistic_attribute) { described_class.statistic_attribute } + + context 'when creating' do + before do + statistic_source.send("#{statistic_attribute}=", 10) + end + + it 'schedules a statistic refresh' do + expect(Groups::UpdateStatisticsWorker) + .to receive(:perform_async) + + statistic_source.save! + end + end + + context 'when updating' do + before do + statistic_source.save! + + expect(statistic_source).to be_persisted + end + + context 'when the statistic attribute has not changed' do + it 'does not schedule a statistic refresh' do + expect(Groups::UpdateStatisticsWorker) + .not_to receive(:perform_async) + + statistic_source.update!(file_name: 'new-file-name.txt') + end + end + + context 'when the statistic attribute has changed' do + it 'schedules a statistic refresh' do + expect(Groups::UpdateStatisticsWorker) + .to receive(:perform_async) + + statistic_source.update!(statistic_attribute => 20) + end + end + end + + context 'when deleting' do + it 'schedules a statistic refresh' do + expect(Groups::UpdateStatisticsWorker) + .to receive(:perform_async) + + statistic_source.destroy! + end + end +end |
