From e4fbd94cf5eba0b103bd97627d822f2014dfe474 Mon Sep 17 00:00:00 2001 From: Victor Zagorodny Date: Wed, 28 Aug 2019 14:26:42 +0000 Subject: Update CE files for GSD projects filter A new param with_security_reports was added to GET /groups/:id/projects API and the code to support this logic in GroupProjectsFinder and Project model. Also, a DB index was added to ci_job_artifacts table to speed up the search of security reports artifacts for projects --- app/finders/group_projects_finder.rb | 12 ++++---- app/models/ci/job_artifact.rb | 2 ++ app/models/project.rb | 1 + ...artifacts_on_project_id_for_security_reports.rb | 22 +++++++++++++++ db/schema.rb | 3 +- doc/api/groups.md | 1 + .../security_dashboard/index.md | 3 ++ lib/api/groups.rb | 1 + lib/api/helpers.rb | 33 ++++++++++++++-------- lib/api/helpers/groups_helpers.rb | 7 +++++ .../group_projects_finder_shared_contexts.rb | 3 +- 11 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 db/migrate/20190828083843_add_index_to_ci_job_artifacts_on_project_id_for_security_reports.rb diff --git a/app/finders/group_projects_finder.rb b/app/finders/group_projects_finder.rb index 4155b6af8da..5e0dbbfca2e 100644 --- a/app/finders/group_projects_finder.rb +++ b/app/finders/group_projects_finder.rb @@ -23,8 +23,12 @@ class GroupProjectsFinder < ProjectsFinder attr_reader :group, :options def initialize(group:, params: {}, options: {}, current_user: nil, project_ids_relation: nil) - super(params: params, current_user: current_user, project_ids_relation: project_ids_relation) - @group = group + super( + params: params, + current_user: current_user, + project_ids_relation: project_ids_relation + ) + @group = group @options = options end @@ -84,15 +88,13 @@ class GroupProjectsFinder < ProjectsFinder options.fetch(:include_subgroups, false) end - # rubocop: disable CodeReuse/ActiveRecord def owned_projects if include_subgroups? - Project.where(namespace_id: group.self_and_descendants.select(:id)) + Project.for_group_and_its_subgroups(group) else group.projects end end - # rubocop: enable CodeReuse/ActiveRecord def shared_projects group.shared_projects diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index e132cb045e2..b4497d8af09 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -87,6 +87,8 @@ module Ci scope :expired, -> (limit) { where('expire_at < ?', Time.now).limit(limit) } + scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } + delegate :filename, :exists?, :open, to: :file enum file_type: { diff --git a/app/models/project.rb b/app/models/project.rb index 66d5286196e..c67c5c7bc8c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -497,6 +497,7 @@ class Project < ApplicationRecord # We require an alias to the project_mirror_data_table in order to use import_state in our queries scope :joins_import_state, -> { joins("INNER JOIN project_mirror_data import_state ON import_state.project_id = projects.id") } scope :for_group, -> (group) { where(group: group) } + scope :for_group_and_its_subgroups, ->(group) { where(namespace_id: group.self_and_descendants.select(:id)) } class << self # Searches for a list of projects based on the query given in `query`. diff --git a/db/migrate/20190828083843_add_index_to_ci_job_artifacts_on_project_id_for_security_reports.rb b/db/migrate/20190828083843_add_index_to_ci_job_artifacts_on_project_id_for_security_reports.rb new file mode 100644 index 00000000000..5253f25aab4 --- /dev/null +++ b/db/migrate/20190828083843_add_index_to_ci_job_artifacts_on_project_id_for_security_reports.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddIndexToCiJobArtifactsOnProjectIdForSecurityReports < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_job_artifacts, + :project_id, + name: "index_ci_job_artifacts_on_project_id_for_security_reports", + where: "file_type IN (5, 6, 7, 8)" + end + + def down + remove_concurrent_index :ci_job_artifacts, + :project_id, + name: "index_ci_job_artifacts_on_project_id_for_security_reports" + end +end diff --git a/db/schema.rb b/db/schema.rb index 6c6c2796b9a..88a7ff77ab4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_20_163320) do +ActiveRecord::Schema.define(version: 2019_08_28_083843) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -658,6 +658,7 @@ ActiveRecord::Schema.define(version: 2019_08_20_163320) do t.index ["file_store"], name: "index_ci_job_artifacts_on_file_store" t.index ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id" + t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id_for_security_reports", where: "(file_type = ANY (ARRAY[5, 6, 7, 8]))" end create_table "ci_job_variables", force: :cascade do |t| diff --git a/doc/api/groups.md b/doc/api/groups.md index 0d500f783aa..d7f5b1b463b 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -158,6 +158,7 @@ Parameters: | `with_shared` | boolean | no | Include projects shared to this group. Default is `true` | | `include_subgroups` | boolean | no | Include projects in subgroups of this group. Default is `false` | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (admins only) | +| `with_security_reports` | boolean | no | **(ULTIMATE)** Return only projects that have security reports artifacts present in any of their builds. This means "projects with security reports enabled". Default is `false` | Example response: diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index ac8c1ac0354..314f7c1766f 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -62,6 +62,9 @@ Once you're on the dashboard, at the top you should see a series of filters for: - Report type - Project +NOTE: **Note:** +The dashboard only shows projects with [security reports](#supported-reports) enabled in a group. + ![dashboard with action buttons and metrics](img/group_security_dashboard.png) Selecting one or more filters will filter the results in this page. diff --git a/lib/api/groups.rb b/lib/api/groups.rb index f545f33c06b..0bcd09d3977 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -216,6 +216,7 @@ module API use :pagination use :with_custom_attributes + use :optional_projects_params end get ":id/projects" do projects = find_group_projects(params) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1aa6dc44bf7..5755f4b8d74 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -416,17 +416,7 @@ module API # rubocop: enable CodeReuse/ActiveRecord def project_finder_params - finder_params = { without_deleted: true } - finder_params[:owned] = true if params[:owned].present? - finder_params[:non_public] = true if params[:membership].present? - finder_params[:starred] = true if params[:starred].present? - finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] - finder_params[:archived] = archived_param unless params[:archived].nil? - finder_params[:search] = params[:search] if params[:search] - finder_params[:user] = params.delete(:user) if params[:user] - finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] - finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] - finder_params + project_finder_params_ce.merge(project_finder_params_ee) end # file helpers @@ -461,6 +451,27 @@ module API end end + protected + + def project_finder_params_ce + finder_params = { without_deleted: true } + finder_params[:owned] = true if params[:owned].present? + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = archived_param unless params[:archived].nil? + finder_params[:search] = params[:search] if params[:search] + finder_params[:user] = params.delete(:user) if params[:user] + finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] + finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] + finder_params + end + + # Overridden in EE + def project_finder_params_ee + {} + end + private # rubocop:disable Gitlab/ModuleWithInstanceVariables diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index 2c33d79f6c8..6af12828ca5 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -28,6 +28,13 @@ module API use :optional_params_ce use :optional_params_ee end + + params :optional_projects_params_ee do + end + + params :optional_projects_params do + use :optional_projects_params_ee + end end end end diff --git a/spec/support/shared_contexts/finders/group_projects_finder_shared_contexts.rb b/spec/support/shared_contexts/finders/group_projects_finder_shared_contexts.rb index 38f6011646e..e7fee7239fc 100644 --- a/spec/support/shared_contexts/finders/group_projects_finder_shared_contexts.rb +++ b/spec/support/shared_contexts/finders/group_projects_finder_shared_contexts.rb @@ -6,9 +6,10 @@ RSpec.shared_context 'GroupProjectsFinder context' do let(:group) { create(:group) } let(:subgroup) { create(:group, parent: group) } let(:current_user) { create(:user) } + let(:params) { {} } let(:options) { {} } - let(:finder) { described_class.new(group: group, current_user: current_user, options: options) } + let(:finder) { described_class.new(group: group, current_user: current_user, params: params, options: options) } let!(:public_project) { create(:project, :public, group: group, path: '1') } let!(:private_project) { create(:project, :private, group: group, path: '2') } -- cgit v1.2.1