From 762ede72cfd597c5d7f4224087c67ecc83d751f8 Mon Sep 17 00:00:00 2001 From: Matt Kasa Date: Thu, 5 Sep 2019 15:00:48 -0700 Subject: Add Cloud Run on GKE feature to cluster creation - Permits cloud_run parameter in ClustersController#create - Enables httpLoadBalancing, istioConfig, and cloudRunConfig in Gcp ProvisionService if cloud_run is enabled - Add `Enable Cloud Run on GKE` checkbox to cluster create page - Add `Enable Cloud Run on GKE` checkbox to cluster details - Default knative to installed for cloud_run clusters - Make knative not uninstallable for cloud_run clusters - Update project clusters docs with entry about Cloud Run - Update tests and add new test for cluster with cloud_run enabled - Add Cloud Run on GKE strings to translations - Add spec that will fail when google-api-client has been upgraded - Add concept of installed-via to applications frontend Relates to https://gitlab.com/gitlab-org/gitlab-ce/issues/59370 --- app/assets/javascripts/clusters/clusters_bundle.js | 6 ++ .../clusters/components/application_row.vue | 19 +++++ .../clusters/components/applications.vue | 14 +++- .../javascripts/clusters/stores/clusters_store.js | 7 ++ app/controllers/clusters/clusters_controller.rb | 1 + app/helpers/clusters_helper.rb | 4 + app/models/clusters/applications/knative.rb | 13 +++ app/services/clusters/gcp/provision_service.rb | 7 +- app/views/clusters/clusters/gcp/_form.html.haml | 7 ++ app/views/clusters/clusters/show.html.haml | 2 + .../unreleased/59370-enable-cloud-run-on-gke.yml | 5 ++ config/initializers/google_api_client.rb | 17 ++++ doc/user/project/clusters/index.md | 10 +++ lib/google_api/cloud_platform/client.rb | 60 +++++++++----- locale/gitlab.pot | 6 ++ spec/factories/clusters/clusters.rb | 4 + spec/factories/clusters/providers/gcp.rb | 4 + .../clusters/stores/clusters_store_spec.js | 1 + spec/initializers/google_api_client_spec.rb | 17 ++++ spec/lib/google_api/cloud_platform/client_spec.rb | 95 ++++++++++++---------- spec/models/clusters/applications/knative_spec.rb | 7 ++ spec/support/google_api/cloud_platform_helpers.rb | 2 +- 22 files changed, 238 insertions(+), 70 deletions(-) create mode 100644 changelogs/unreleased/59370-enable-cloud-run-on-gke.yml create mode 100644 config/initializers/google_api_client.rb create mode 100644 spec/initializers/google_api_client_spec.rb diff --git a/app/assets/javascripts/clusters/clusters_bundle.js b/app/assets/javascripts/clusters/clusters_bundle.js index 1f213d5aaf2..328a72a822b 100644 --- a/app/assets/javascripts/clusters/clusters_bundle.js +++ b/app/assets/javascripts/clusters/clusters_bundle.js @@ -50,6 +50,8 @@ export default class Clusters { clustersHelpPath, deployBoardsHelpPath, clusterId, + cloudRun, + cloudRunHelpPath, } = document.querySelector('.js-edit-cluster-form').dataset; this.clusterId = clusterId; @@ -64,11 +66,13 @@ export default class Clusters { environmentsHelpPath, clustersHelpPath, deployBoardsHelpPath, + cloudRunHelpPath, ); this.store.setManagePrometheusPath(managePrometheusPath); this.store.updateStatus(clusterStatus); this.store.updateStatusReason(clusterStatusReason); this.store.updateRbac(hasRbac); + this.store.updateCloudRun(cloudRun); this.service = new ClustersService({ endpoint: statusPath, installHelmEndpoint: installHelmPath, @@ -138,6 +142,8 @@ export default class Clusters { managePrometheusPath: this.state.managePrometheusPath, ingressDnsHelpPath: this.state.ingressDnsHelpPath, rbac: this.state.rbac, + cloudRun: this.state.cloudRun, + cloudRunHelpPath: this.state.cloudRunHelpPath, }, }); }, diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue index 64364092016..56480f17932 100644 --- a/app/assets/javascripts/clusters/components/application_row.vue +++ b/app/assets/javascripts/clusters/components/application_row.vue @@ -73,6 +73,14 @@ export default { required: false, default: false, }, + installedVia: { + type: String, + required: false, + }, + installedViaLink: { + type: String, + required: false, + }, installFailed: { type: Boolean, required: false, @@ -311,6 +319,17 @@ export default { > {{ title }} + + installed via + {{ installedVia }} + {{ installedVia }} +

diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue index b6da572b201..46c64453333 100644 --- a/app/assets/javascripts/clusters/components/applications.vue +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -64,6 +64,16 @@ export default { required: false, default: false, }, + cloudRun: { + type: Boolean, + required: false, + default: false, + }, + cloudRunHelpPath: { + type: String, + required: false, + default: '', + }, }, data: () => ({ elasticsearchLogo, @@ -467,6 +477,8 @@ export default { :request-status="applications.knative.requestStatus" :request-reason="applications.knative.requestReason" :installed="applications.knative.installed" + :installed-via="cloudRun ? 'Cloud Run' : null" + :installed-via-link="cloudRun ? cloudRunHelpPath : null" :install-failed="applications.knative.installFailed" :install-application-request-params="{ hostname: applications.knative.hostname }" :uninstallable="applications.knative.uninstallable" @@ -500,7 +512,7 @@ export default {

[:installed] do |application| + break if application.cloud_run_cluster? + application.run_after_commit do ClusterWaitForIngressIpAddressWorker.perform_in( FETCH_IP_ADDRESS_DELAY, application.name, application.id) @@ -47,6 +51,10 @@ module Clusters { "domain" => hostname }.to_yaml end + def allowed_to_uninstall? + !cloud_run_cluster? + end + def install_command Gitlab::Kubernetes::Helm::InstallCommand.new( name: name, @@ -63,6 +71,7 @@ module Clusters return unless installed? return if external_ip return if external_hostname + return if cloud_run_cluster? ClusterWaitForIngressIpAddressWorker.perform_async(name, id) end @@ -81,6 +90,10 @@ module Clusters ) end + def cloud_run_cluster? + cluster&.gcp? && cluster&.provider_gcp&.cloud_run? + end + private def delete_knative_services_and_metrics diff --git a/app/services/clusters/gcp/provision_service.rb b/app/services/clusters/gcp/provision_service.rb index 80040511ec2..7dc2d3c32f1 100644 --- a/app/services/clusters/gcp/provision_service.rb +++ b/app/services/clusters/gcp/provision_service.rb @@ -3,6 +3,8 @@ module Clusters module Gcp class ProvisionService + CLOUD_RUN_ADDONS = %i[http_load_balancing istio_config cloud_run_config].freeze + attr_reader :provider def execute(provider) @@ -22,13 +24,16 @@ module Clusters private def get_operation_id + enable_addons = provider.cloud_run? ? CLOUD_RUN_ADDONS : [] + operation = provider.api_client.projects_zones_clusters_create( provider.gcp_project_id, provider.zone, provider.cluster.name, provider.num_nodes, machine_type: provider.machine_type, - legacy_abac: provider.legacy_abac + legacy_abac: provider.legacy_abac, + enable_addons: enable_addons ) unless operation.status == 'PENDING' || operation.status == 'RUNNING' diff --git a/app/views/clusters/clusters/gcp/_form.html.haml b/app/views/clusters/clusters/gcp/_form.html.haml index 4d3e3359ea0..196ad422766 100644 --- a/app/views/clusters/clusters/gcp/_form.html.haml +++ b/app/views/clusters/clusters/gcp/_form.html.haml @@ -65,6 +65,13 @@ %p.form-text.text-muted = s_('ClusterIntegration|Learn more about %{help_link_start_machine_type}machine types%{help_link_end} and %{help_link_start_pricing}pricing%{help_link_end}.').html_safe % { help_link_start_machine_type: help_link_start % { url: machine_type_link_url }, help_link_start_pricing: help_link_start % { url: pricing_link_url }, help_link_end: help_link_end } + .form-group + = provider_gcp_field.check_box :cloud_run, { label: s_('ClusterIntegration|Enable Cloud Run on GKE (beta)'), + label_class: 'label-bold' } + .form-text.text-muted + = s_('ClusterIntegration|Uses the Cloud Run, Istio, and HTTP Load Balancing addons for this cluster.') + = link_to _('More information'), help_page_path('user/project/clusters/index.md', anchor: 'cloud-run-on-gke'), target: '_blank' + .form-group = field.check_box :managed, { label: s_('ClusterIntegration|GitLab-managed cluster'), label_class: 'label-bold' } diff --git a/app/views/clusters/clusters/show.html.haml b/app/views/clusters/clusters/show.html.haml index 6052e5d96f2..62d538e64dd 100644 --- a/app/views/clusters/clusters/show.html.haml +++ b/app/views/clusters/clusters/show.html.haml @@ -18,6 +18,7 @@ update_knative_path: clusterable.update_applications_cluster_path(@cluster, :knative), toggle_status: @cluster.enabled? ? 'true': 'false', has_rbac: has_rbac_enabled?(@cluster) ? 'true': 'false', + cloud_run: cloud_run?(@cluster) ? 'true': 'false', cluster_type: @cluster.cluster_type, cluster_status: @cluster.status_name, cluster_status_reason: @cluster.status_reason, @@ -27,6 +28,7 @@ environments_help_path: help_page_path('ci/environments', anchor: 'defining-environments'), clusters_help_path: help_page_path('user/project/clusters/index.md', anchor: 'deploying-to-a-kubernetes-cluster'), deploy_boards_help_path: help_page_path('user/project/deploy_boards.html', anchor: 'enabling-deploy-boards'), + cloud_run_help_path: help_page_path('user/project/clusters/index.md', anchor: 'cloud-run-on-gke'), manage_prometheus_path: manage_prometheus_path, cluster_id: @cluster.id } } diff --git a/changelogs/unreleased/59370-enable-cloud-run-on-gke.yml b/changelogs/unreleased/59370-enable-cloud-run-on-gke.yml new file mode 100644 index 00000000000..7e137c06074 --- /dev/null +++ b/changelogs/unreleased/59370-enable-cloud-run-on-gke.yml @@ -0,0 +1,5 @@ +--- +title: Enable Cloud Run on GKE cluster creation +merge_request: 31790 +author: +type: added diff --git a/config/initializers/google_api_client.rb b/config/initializers/google_api_client.rb new file mode 100644 index 00000000000..611726a20c7 --- /dev/null +++ b/config/initializers/google_api_client.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +# +# google-api-client >= 0.26.0 supports enabling CloudRun and Istio during +# cluster creation, but fog-google currently hard deps on '~> 0.23.0', which +# prevents us from upgrading. We are injecting these options as hashes below +# as a workaround until this is resolved. +# +# This can be removed once fog-google and google-api-client can be upgraded. +# See https://gitlab.com/gitlab-org/gitlab-ce/issues/66630 for more details. +# + +require 'google/apis/container_v1beta1' + +Google::Apis::ContainerV1beta1::AddonsConfig::Representation.tap do |representation| + representation.hash :cloud_run_config, as: 'cloudRunConfig' + representation.hash :istio_config, as: 'istioConfig' +end diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 3bde0a375c6..0bf8884a169 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -154,6 +154,7 @@ new Kubernetes cluster to your project: - **Number of nodes** - Enter the number of nodes you wish the cluster to have. - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-types) of the Virtual Machine instance that the cluster will be based on. + - **Enable Cloud Run on GKE (beta)** - Check this if you want to use Cloud Run on GKE for this cluster. See the [Cloud Run on GKE section](#cloud-run-on-gke) for more information. - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](#gitlab-managed-clusters) for more information. 1. Finally, click the **Create Kubernetes cluster** button. @@ -339,6 +340,15 @@ functionalities needed to successfully build and deploy a containerized application. Bear in mind that the same credentials are used for all the applications running on the cluster. +### Cloud Run on GKE + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31790) in GitLab 12.3. + +You can choose to use Cloud Run on GKE in place of installing Knative and Istio +separately after the cluster has been created. This means that Cloud Run +(Knative), Istio, and HTTP Load Balancing will be enabled on the cluster at +create time and cannot be installed or uninstalled separately. + ### GitLab-managed clusters > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22011) in GitLab 11.5. diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index b5f99ea012b..cb5448b91da 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -2,6 +2,7 @@ require 'google/apis/compute_v1' require 'google/apis/container_v1' +require 'google/apis/container_v1beta1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' @@ -53,30 +54,14 @@ module GoogleApi service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end - def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:) - service = Google::Apis::ContainerV1::ContainerService.new + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:, enable_addons: []) + service = Google::Apis::ContainerV1beta1::ContainerService.new service.authorization = access_token - request_body = Google::Apis::ContainerV1::CreateClusterRequest.new( - { - "cluster": { - "name": cluster_name, - "initial_node_count": cluster_size, - "node_config": { - "machine_type": machine_type - }, - "master_auth": { - "username": CLUSTER_MASTER_AUTH_USERNAME, - "client_certificate_config": { - issue_client_certificate: true - } - }, - "legacy_abac": { - "enabled": legacy_abac - } - } - } - ) + cluster_options = make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac) + enable_addons!(cluster_options, enable_addons) + + request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(cluster_options) service.create_cluster(project_id, zone, request_body, options: user_agent_header) end @@ -95,6 +80,37 @@ module GoogleApi private + def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac) + { + cluster: { + name: cluster_name, + initial_node_count: cluster_size, + node_config: { + machine_type: machine_type + }, + master_auth: { + username: CLUSTER_MASTER_AUTH_USERNAME, + client_certificate_config: { + issue_client_certificate: true + } + }, + legacy_abac: { + enabled: legacy_abac + } + } + } + end + + def enable_addons!(cluster_options, enable_addons) + (cluster_options[:cluster][:addons_config] ||= {}).tap do |addons_config| + enable_addons.each do |addon| + (addons_config[addon] ||= {}).tap do |addon_config| + addon_config[:disabled] = false + end + end + end + end + def token_life_time(expires_at) DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index d367503598a..53978080b80 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2584,6 +2584,9 @@ msgstr "" msgid "ClusterIntegration|Did you know?" msgstr "" +msgid "ClusterIntegration|Enable Cloud Run on GKE (beta)" +msgstr "" + msgid "ClusterIntegration|Enable or disable GitLab's connection to your Kubernetes cluster." msgstr "" @@ -2920,6 +2923,9 @@ msgstr "" msgid "ClusterIntegration|Update failed. Please check the logs and try again." msgstr "" +msgid "ClusterIntegration|Uses the Cloud Run, Istio, and HTTP Load Balancing addons for this cluster." +msgstr "" + msgid "ClusterIntegration|Validating project billing status" msgstr "" diff --git a/spec/factories/clusters/clusters.rb b/spec/factories/clusters/clusters.rb index d294e6d055e..29aea5e403e 100644 --- a/spec/factories/clusters/clusters.rb +++ b/spec/factories/clusters/clusters.rb @@ -58,6 +58,10 @@ FactoryBot.define do platform_kubernetes factory: [:cluster_platform_kubernetes, :configured, :rbac_disabled] end + trait :cloud_run_enabled do + provider_gcp factory: [:cluster_provider_gcp, :created, :cloud_run_enabled] + end + trait :disabled do enabled false end diff --git a/spec/factories/clusters/providers/gcp.rb b/spec/factories/clusters/providers/gcp.rb index 22462651b6a..7fdcdebad34 100644 --- a/spec/factories/clusters/providers/gcp.rb +++ b/spec/factories/clusters/providers/gcp.rb @@ -34,5 +34,9 @@ FactoryBot.define do trait :abac_enabled do legacy_abac true end + + trait :cloud_run_enabled do + cloud_run true + end end end diff --git a/spec/frontend/clusters/stores/clusters_store_spec.js b/spec/frontend/clusters/stores/clusters_store_spec.js index 98077498998..e4dc22b4883 100644 --- a/spec/frontend/clusters/stores/clusters_store_spec.js +++ b/spec/frontend/clusters/stores/clusters_store_spec.js @@ -52,6 +52,7 @@ describe('Clusters Store', () => { helpPath: null, ingressHelpPath: null, environmentsHelpPath: null, + cloudRun: false, clustersHelpPath: null, deployBoardsHelpPath: null, status: mockResponseData.status, diff --git a/spec/initializers/google_api_client_spec.rb b/spec/initializers/google_api_client_spec.rb new file mode 100644 index 00000000000..44a1bc0836c --- /dev/null +++ b/spec/initializers/google_api_client_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe './config/initializers/google_api_client.rb' do + subject { Google::Apis::ContainerV1beta1 } + + it 'is needed' do |example| + is_expected.not_to be_const_defined(:CloudRunConfig), + <<-MSG.strip_heredoc + The google-api-client gem has been upgraded! + Remove: + #{example.example_group.description} + #{example.file_path} + MSG + end +end diff --git a/spec/lib/google_api/cloud_platform/client_spec.rb b/spec/lib/google_api/cloud_platform/client_spec.rb index c24998d32f8..4c223dffd2f 100644 --- a/spec/lib/google_api/cloud_platform/client_spec.rb +++ b/spec/lib/google_api/cloud_platform/client_spec.rb @@ -68,7 +68,7 @@ describe GoogleApi::CloudPlatform::Client do describe '#projects_zones_clusters_create' do subject do client.projects_zones_clusters_create( - project_id, zone, cluster_name, cluster_size, machine_type: machine_type, legacy_abac: legacy_abac) + project_id, zone, cluster_name, cluster_size, machine_type: machine_type, legacy_abac: legacy_abac, enable_addons: enable_addons) end let(:project_id) { 'project-123' } @@ -77,39 +77,48 @@ describe GoogleApi::CloudPlatform::Client do let(:cluster_size) { 1 } let(:machine_type) { 'n1-standard-2' } let(:legacy_abac) { true } - let(:create_cluster_request_body) { double('Google::Apis::ContainerV1::CreateClusterRequest') } + let(:enable_addons) { [] } + let(:addons_config) do + enable_addons.each_with_object({}) do |addon, hash| + hash[addon] = { disabled: false } + end + end + let(:cluster_options) do + { + cluster: { + name: cluster_name, + initial_node_count: cluster_size, + node_config: { + machine_type: machine_type + }, + master_auth: { + username: 'admin', + client_certificate_config: { + issue_client_certificate: true + } + }, + legacy_abac: { + enabled: legacy_abac + }, + addons_config: addons_config + } + } + end + let(:create_cluster_request_body) { double('Google::Apis::ContainerV1beta1::CreateClusterRequest') } let(:operation) { double } before do - allow_any_instance_of(Google::Apis::ContainerV1::ContainerService) + allow_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) .to receive(:create_cluster).with(any_args) .and_return(operation) end it 'sets corresponded parameters' do - expect_any_instance_of(Google::Apis::ContainerV1::ContainerService) + expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) - expect(Google::Apis::ContainerV1::CreateClusterRequest) - .to receive(:new).with( - { - "cluster": { - "name": cluster_name, - "initial_node_count": cluster_size, - "node_config": { - "machine_type": machine_type - }, - "master_auth": { - "username": "admin", - "client_certificate_config": { - issue_client_certificate: true - } - }, - "legacy_abac": { - "enabled": true - } - } - } ).and_return(create_cluster_request_body) + expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) + .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) expect(subject).to eq operation end @@ -118,29 +127,25 @@ describe GoogleApi::CloudPlatform::Client do let(:legacy_abac) { false } it 'sets corresponded parameters' do - expect_any_instance_of(Google::Apis::ContainerV1::ContainerService) + expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) + .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) + + expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) + .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) + + expect(subject).to eq operation + end + end + + context 'create with enable_addons for cloud_run' do + let(:enable_addons) { [:http_load_balancing, :istio_config, :cloud_run_config] } + + it 'sets corresponded parameters' do + expect_any_instance_of(Google::Apis::ContainerV1beta1::ContainerService) .to receive(:create_cluster).with(project_id, zone, create_cluster_request_body, options: user_agent_options) - expect(Google::Apis::ContainerV1::CreateClusterRequest) - .to receive(:new).with( - { - "cluster": { - "name": cluster_name, - "initial_node_count": cluster_size, - "node_config": { - "machine_type": machine_type - }, - "master_auth": { - "username": "admin", - "client_certificate_config": { - issue_client_certificate: true - } - }, - "legacy_abac": { - "enabled": false - } - } - } ).and_return(create_cluster_request_body) + expect(Google::Apis::ContainerV1beta1::CreateClusterRequest) + .to receive(:new).with(cluster_options).and_return(create_cluster_request_body) expect(subject).to eq operation end diff --git a/spec/models/clusters/applications/knative_spec.rb b/spec/models/clusters/applications/knative_spec.rb index 3825994b733..16247026e34 100644 --- a/spec/models/clusters/applications/knative_spec.rb +++ b/spec/models/clusters/applications/knative_spec.rb @@ -16,6 +16,13 @@ describe Clusters::Applications::Knative do allow(ClusterWaitForIngressIpAddressWorker).to receive(:perform_async) end + describe 'when cloud run is enabled' do + let(:cluster) { create(:cluster, :provided_by_gcp, :cloud_run_enabled) } + let(:knative_cloud_run) { create(:clusters_applications_knative, cluster: cluster) } + + it { expect(knative_cloud_run).to be_not_installable } + end + describe 'when rbac is not enabled' do let(:cluster) { create(:cluster, :provided_by_gcp, :rbac_disabled) } let(:knative_no_rbac) { create(:clusters_applications_knative, cluster: cluster) } diff --git a/spec/support/google_api/cloud_platform_helpers.rb b/spec/support/google_api/cloud_platform_helpers.rb index a1328ef0d13..38ffca8c5ae 100644 --- a/spec/support/google_api/cloud_platform_helpers.rb +++ b/spec/support/google_api/cloud_platform_helpers.rb @@ -65,7 +65,7 @@ module GoogleApi end def cloud_platform_create_cluster_url(project_id, zone) - "https://container.googleapis.com/v1/projects/#{project_id}/zones/#{zone}/clusters" + "https://container.googleapis.com/v1beta1/projects/#{project_id}/zones/#{zone}/clusters" end def cloud_platform_get_zone_operation_url(project_id, zone, operation_id) -- cgit v1.2.1