Add GitHub & Gitea importer projects filteringgeorgekoltsov/add-github-importer-filtering

Backend:
  - Add new filter param
  - Add sanitization of filter param
  - Add actual filtering of repos/projects collection

Frontend:
  - Add new input field on the UI
  - Add new filter to Import Projects Table state
  - Add realtime changes endpoint filtering
  - Add throttling of filtering http requests

- Update documentation & screenshots

18 files changed, 219 insertions, 27 deletions

diff --git a/app/assets/javascripts/import_projects/components/import_projects_table.vue b/app/assets/javascripts/import_projects/components/import_projects_table.vue
index 00eb0afb3bf..969ff1c5e04 100644
--- a/app/assets/javascripts/import_projects/components/import_projects_table.vue
+++ b/app/assets/javascripts/import_projects/components/import_projects_table.vue
@@ -1,4 +1,5 @@
 <script>
+import _ from 'underscore';
 import { mapActions, mapState, mapGetters } from 'vuex';
 import { GlLoadingIcon } from '@gitlab/ui';
 import LoadingButton from '~/vue_shared/components/loading_button.vue';
@@ -7,6 +8,8 @@ import ImportedProjectTableRow from './imported_project_table_row.vue';
 import ProviderRepoTableRow from './provider_repo_table_row.vue';
 import eventHub from '../event_hub';
 
+const reposFetchThrottleDelay = 1000;
+
 export default {
   name: 'ImportProjectsTable',
   components: {
@@ -23,11 +26,11 @@ export default {
   },
 
   computed: {
-    ...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos']),
+    ...mapState(['importedProjects', 'providerRepos', 'isLoadingRepos', 'filter']),
     ...mapGetters(['isImportingAnyRepo', 'hasProviderRepos', 'hasImportedProjects']),
 
     emptyStateText() {
-      return sprintf(__('No %{providerTitle} repositories available to import'), {
+      return sprintf(__('No %{providerTitle} repositories found'), {
         providerTitle: this.providerTitle,
       });
     },
@@ -35,6 +38,16 @@ export default {
     fromHeaderText() {
       return sprintf(__('From %{providerTitle}'), { providerTitle: this.providerTitle });
     },
+
+    filter: {
+      get() {
+        return this.$store.state.filter;
+      },
+
+      set(value) {
+        this.$store.state.filter = value;
+      },
+    },
   },
 
   mounted() {
@@ -47,21 +60,38 @@ export default {
   },
 
   methods: {
-    ...mapActions(['fetchRepos', 'fetchJobs', 'stopJobsPolling', 'clearJobsEtagPoll']),
+    ...mapActions([
+      'fetchRepos',
+      'fetchReposFiltered',
+      'fetchJobs',
+      'stopJobsPolling',
+      'clearJobsEtagPoll',
+      'setFilter',
+    ]),
 
     importAll() {
       eventHub.$emit('importAll');
     },
+
+    handleFilterInput({ target }) {
+      this.setFilter(target.value);
+    },
+
+    throttledFetchRepos: _.throttle(function fetch() {
+      eventHub.$off('importAll');
+      this.fetchRepos();
+    }, reposFetchThrottleDelay),
   },
 };
 </script>
 
 <template>
   <div>
+    <p class="light text-nowrap mt-2">
+      {{ s__('ImportProjects|Select the projects you want to import') }}
+    </p>
+
     <div class="d-flex justify-content-between align-items-end flex-wrap mb-3">
-      <p class="light text-nowrap mt-2 my-sm-0">
-        {{ s__('ImportProjects|Select the projects you want to import') }}
-      </p>
       <loading-button
         container-class="btn btn-success js-import-all"
         :loading="isImportingAnyRepo"
@@ -70,6 +100,19 @@ export default {
         type="button"
         @click="importAll"
       />
+      <form novalidate @submit.prevent>
+        <input
+          :value="filter"
+          data-qa-selector="githubish_import_filter_field"
+          class="form-control"
+          name="filter"
+          :placeholder="__('Filter your projects by name')"
+          autofocus
+          size="40"
+          @input="handleFilterInput($event)"
+          @keyup.enter="throttledFetchRepos"
+        />
+      </form>
     </div>
     <gl-loading-icon
       v-if="isLoadingRepos"
diff --git a/app/assets/javascripts/import_projects/index.js b/app/assets/javascripts/import_projects/index.js
index 2d99d716609..b069dcb7766 100644
--- a/app/assets/javascripts/import_projects/index.js
+++ b/app/assets/javascripts/import_projects/index.js
@@ -38,7 +38,7 @@ export default function mountImportProjectsTable(mountElement) {
     },
 
     methods: {
-      ...mapActions(['setInitialData']),
+      ...mapActions(['setInitialData', 'setFilter']),
     },
 
     render(createElement) {
diff --git a/app/assets/javascripts/import_projects/store/actions.js b/app/assets/javascripts/import_projects/store/actions.js
index c44500937cc..350c7718b03 100644
--- a/app/assets/javascripts/import_projects/store/actions.js
+++ b/app/assets/javascripts/import_projects/store/actions.js
@@ -5,6 +5,7 @@ import Poll from '~/lib/utils/poll';
 import createFlash from '~/flash';
 import { s__, sprintf } from '~/locale';
 import axios from '~/lib/utils/axios_utils';
+import { jobsPathWithFilter, reposPathWithFilter } from "./getters";
 
 let eTagPoll;
 
@@ -19,16 +20,20 @@ export const restartJobsPolling = () => {
 };
 
 export const setInitialData = ({ commit }, data) => commit(types.SET_INITIAL_DATA, data);
+export const setFilter = ({ commit }, filter) => commit(types.SET_FILTER, filter);
 
 export const requestRepos = ({ commit }, repos) => commit(types.REQUEST_REPOS, repos);
 export const receiveReposSuccess = ({ commit }, repos) =>
   commit(types.RECEIVE_REPOS_SUCCESS, repos);
 export const receiveReposError = ({ commit }) => commit(types.RECEIVE_REPOS_ERROR);
 export const fetchRepos = ({ state, dispatch }) => {
+  dispatch('stopJobsPolling');
   dispatch('requestRepos');
 
+  const { provider } = state;
+
   return axios
-    .get(state.reposPath)
+    .get(reposPathWithFilter(state))
     .then(({ data }) =>
       dispatch('receiveReposSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
     )
@@ -36,7 +41,7 @@ export const fetchRepos = ({ state, dispatch }) => {
     .catch(() => {
       createFlash(
         sprintf(s__('ImportProjects|Requesting your %{provider} repositories failed'), {
-          provider: state.provider,
+          provider,
         }),
       );
 
@@ -77,16 +82,23 @@ export const fetchImport = ({ state, dispatch }, { newName, targetNamespace, rep
 export const receiveJobsSuccess = ({ commit }, updatedProjects) =>
   commit(types.RECEIVE_JOBS_SUCCESS, updatedProjects);
 export const fetchJobs = ({ state, dispatch }) => {
-  if (eTagPoll) return;
+  const { filter } = state;
+
+  if (eTagPoll) {
+    stopJobsPolling();
+    clearJobsEtagPoll();
+  }
 
   eTagPoll = new Poll({
     resource: {
-      fetchJobs: () => axios.get(state.jobsPath),
+      fetchJobs: () => axios.get(jobsPathWithFilter(state)),
     },
     method: 'fetchJobs',
     successCallback: ({ data }) =>
       dispatch('receiveJobsSuccess', convertObjectPropsToCamelCase(data, { deep: true })),
-    errorCallback: () => createFlash(s__('ImportProjects|Updating the imported projects failed')),
+    errorCallback: () =>
+      createFlash(s__('ImportProjects|Update of imported projects with realtime changes failed')),
+    data: { filter },
   });
 
   if (!Visibility.hidden()) {
diff --git a/app/assets/javascripts/import_projects/store/getters.js b/app/assets/javascripts/import_projects/store/getters.js
index 727b80765bd..b763cd76eb2 100644
--- a/app/assets/javascripts/import_projects/store/getters.js
+++ b/app/assets/javascripts/import_projects/store/getters.js
@@ -20,3 +20,14 @@ export const isImportingAnyRepo = state => state.reposBeingImported.length > 0;
 export const hasProviderRepos = state => state.providerRepos.length > 0;
 
 export const hasImportedProjects = state => state.importedProjects.length > 0;
+
+export const concatenatedPath = (path, filter) => {
+  if (filter && filter.length > 0) {
+    return path.concat(`?filter=${filter}`);
+  }
+
+  return path;
+};
+
+export const reposPathWithFilter = state => concatenatedPath(state.reposPath, state.filter);
+export const jobsPathWithFilter = state => concatenatedPath(state.jobsPath, state.filter);
diff --git a/app/assets/javascripts/import_projects/store/mutation_types.js b/app/assets/javascripts/import_projects/store/mutation_types.js
index 6ba3fd6f29e..16574f4450f 100644
--- a/app/assets/javascripts/import_projects/store/mutation_types.js
+++ b/app/assets/javascripts/import_projects/store/mutation_types.js
@@ -9,3 +9,5 @@ export const RECEIVE_IMPORT_SUCCESS = 'RECEIVE_IMPORT_SUCCESS';
 export const RECEIVE_IMPORT_ERROR = 'RECEIVE_IMPORT_ERROR';
 
 export const RECEIVE_JOBS_SUCCESS = 'RECEIVE_JOBS_SUCCESS';
+
+export const SET_FILTER = 'SET_FILTER';
diff --git a/app/assets/javascripts/import_projects/store/mutations.js b/app/assets/javascripts/import_projects/store/mutations.js
index b88de0268e7..6c56cfa8298 100644
--- a/app/assets/javascripts/import_projects/store/mutations.js
+++ b/app/assets/javascripts/import_projects/store/mutations.js
@@ -6,6 +6,10 @@ export default {
     Object.assign(state, data);
   },
 
+  [types.SET_FILTER](state, filter) {
+    state.filter = filter;
+  },
+
   [types.REQUEST_REPOS](state) {
     state.isLoadingRepos = true;
   },
diff --git a/app/assets/javascripts/import_projects/store/state.js b/app/assets/javascripts/import_projects/store/state.js
index 637fef6e53c..829f3aa4fbb 100644
--- a/app/assets/javascripts/import_projects/store/state.js
+++ b/app/assets/javascripts/import_projects/store/state.js
@@ -12,4 +12,5 @@ export default () => ({
   isLoadingRepos: false,
   canSelectNamespace: false,
   ciCdOnly: false,
+  filter: '',
 });
diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb
index 1cea0299fb2..cac5654fa57 100644
--- a/app/controllers/import/github_controller.rb
+++ b/app/controllers/import/github_controller.rb
@@ -2,6 +2,7 @@
 
 class Import::GithubController < Import::BaseController
   include ImportHelper
+  include ActionView::Helpers::SanitizeHelper
 
   before_action :verify_import_enabled
   before_action :provider_auth, only: [:status, :realtime_changes, :create]
@@ -55,7 +56,7 @@ class Import::GithubController < Import::BaseController
   def realtime_changes
     Gitlab::PollingInterval.set_header(response, interval: 3_000)
 
-    render json: find_jobs(provider)
+    render json: already_added_projects.to_json(only: [:id], methods: [:import_status])
   end
 
   private
@@ -82,7 +83,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def already_added_projects
-    @already_added_projects ||= find_already_added_projects(provider)
+    @already_added_projects ||= filtered(find_already_added_projects(provider))
   end
 
   def already_added_project_names
@@ -104,7 +105,7 @@ class Import::GithubController < Import::BaseController
   end
 
   def client_repos
-    @client_repos ||= client.repos
+    @client_repos ||= filtered(client.repos)
   end
 
   def verify_import_enabled
@@ -185,6 +186,20 @@ class Import::GithubController < Import::BaseController
   def extra_import_params
     {}
   end
+
+  def sanitized_filter_param
+    @filter ||= sanitize(params[:filter])
+  end
+
+  def filter_attribute
+    :name
+  end
+
+  def filtered(collection)
+    return collection unless sanitized_filter_param
+
+    collection.select { |item| item[filter_attribute].include?(sanitized_filter_param) }
+  end
 end
 
 Import::GithubController.prepend_if_ee('EE::Import::GithubController')
diff --git a/changelogs/unreleased/georgekoltsov-add-github-importer-filtering.yml b/changelogs/unreleased/georgekoltsov-add-github-importer-filtering.yml
new file mode 100644
index 00000000000..8b975e8c9d7
--- /dev/null
+++ b/changelogs/unreleased/georgekoltsov-add-github-importer-filtering.yml
@@ -0,0 +1,5 @@
+---
+title: Add GitHub importer projects filtering
+merge_request: 32596
+author:
+type: added
diff --git a/doc/user/project/import/gitea.md b/doc/user/project/import/gitea.md
index f5746a0fb31..0b9034c821b 100644
--- a/doc/user/project/import/gitea.md
+++ b/doc/user/project/import/gitea.md
@@ -66,10 +66,14 @@ From there, you can see the import statuses of your Gitea repositories.
 - whereas those that are not yet imported will have an **Import** button on the
   right side of the table.
 
-If you want, you can import all your Gitea projects in one go by hitting
-**Import all projects** in the upper left corner.
+You also can:
 
-![Gitea importer page](img/import_projects_from_github_importer.png)
+- Import all your Gitea projects in one go by hitting **Import all projects** in
+  the upper left corner
+- Filter projects by name. If filter is applied, hitting **Import all projects**
+  will only import matched projects
+
+![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png)
 
 ---
 
diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md
index dad53a600dc..0fd724f63ac 100644
--- a/doc/user/project/import/github.md
+++ b/doc/user/project/import/github.md
@@ -115,11 +115,14 @@ your GitHub repositories are listed.
 
 1. By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions,
    you can choose to edit these names before you proceed to import any of them.
-1. Select the **Import** button next to any number of repositories, or select **Import all repositories**.
+1. Select the **Import** button next to any number of repositories, or select **Import all repositories**. Additionally,
+   you can filter projects by name. If filter is applied, **Import all repositories** only imports matched repositories.
 1. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will
    update in realtime or you can return to it later.
 1. Once a repository has been imported, click its GitLab path to open its GitLab URL.
 
+![Github importer page](img/import_projects_from_github_importer_v12_3.png)
+
 ## Mirroring and pipeline status sharing
 
 Depending your GitLab tier, [project mirroring](../../../workflow/repository_mirroring.md) can be set up to keep
diff --git a/doc/user/project/import/img/import_projects_from_gitea_importer_v12_3.png b/doc/user/project/import/img/import_projects_from_gitea_importer_v12_3.png
new file mode 100644
index 00000000000..d8ae1a54851
--- /dev/null
+++ b/doc/user/project/import/img/import_projects_from_gitea_importer_v12_3.png
diff --git a/doc/user/project/import/img/import_projects_from_github_importer.png b/doc/user/project/import/img/import_projects_from_github_importer.png
deleted file mode 100644
index d8effaf6075..00000000000
--- a/doc/user/project/import/img/import_projects_from_github_importer.png
+++ /dev/null
diff --git a/doc/user/project/import/img/import_projects_from_github_importer_v12_3.png b/doc/user/project/import/img/import_projects_from_github_importer_v12_3.png
new file mode 100644
index 00000000000..6a53d9e6d1d
--- /dev/null
+++ b/doc/user/project/import/img/import_projects_from_github_importer_v12_3.png
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 37b589d902d..ab5e30a2526 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -8222,7 +8222,7 @@ msgstr ""
 msgid "ImportProjects|The repository could not be created."
 msgstr ""
 
-msgid "ImportProjects|Updating the imported projects failed"
+msgid "ImportProjects|Update of imported projects with realtime changes failed"
 msgstr ""
 
 msgid "Improve Issue boards"
@@ -10157,7 +10157,7 @@ msgstr ""
 msgid "No %{header} for this request."
 msgstr ""
 
-msgid "No %{providerTitle} repositories available to import"
+msgid "No %{providerTitle} repositories found"
 msgstr ""
 
 msgid "No Epic"
diff --git a/spec/frontend/import_projects/components/import_projects_table_spec.js b/spec/frontend/import_projects/components/import_projects_table_spec.js
index 17a998d0174..708f2758083 100644
--- a/spec/frontend/import_projects/components/import_projects_table_spec.js
+++ b/spec/frontend/import_projects/components/import_projects_table_spec.js
@@ -93,7 +93,7 @@ describe('ImportProjectsTable', () => {
     return vm.$nextTick().then(() => {
       expect(vm.$el.querySelector('.js-loading-button-icon')).toBeNull();
       expect(vm.$el.querySelector('.table')).toBeNull();
-      expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories available to import`);
+      expect(vm.$el.innerText).toMatch(`No ${providerTitle} repositories found`);
     });
   });
 
@@ -182,4 +182,10 @@ describe('ImportProjectsTable', () => {
         expect(vm.$el.querySelector(`.ic-status_${statusObject.icon}`)).not.toBeNull();
       });
   });
+
+  it('renders filtering input field', () => {
+    expect(
+      vm.$el.querySelector('input[data-qa-selector="githubish_import_filter_field"]'),
+    ).not.toBeNull();
+  });
 });
diff --git a/spec/frontend/import_projects/store/actions_spec.js b/spec/frontend/import_projects/store/actions_spec.js
index 6a7b90788dd..340b6f02d93 100644
--- a/spec/frontend/import_projects/store/actions_spec.js
+++ b/spec/frontend/import_projects/store/actions_spec.js
@@ -97,6 +97,7 @@ describe('import_projects store actions', () => {
 
   describe('fetchRepos', () => {
     let mock;
+    const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
 
     beforeEach(() => {
       localState.reposPath = `${TEST_HOST}/endpoint.json`;
@@ -105,8 +106,7 @@ describe('import_projects store actions', () => {
 
     afterEach(() => mock.restore());
 
-    it('dispatches requestRepos and receiveReposSuccess actions on a successful request', done => {
-      const payload = { imported_projects: [{}], provider_repos: [{}], namespaces: [{}] };
+    it('dispatches stopJobsPolling, requestRepos and receiveReposSuccess actions on a successful request', done => {
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, payload);
 
       testAction(
@@ -115,6 +115,7 @@ describe('import_projects store actions', () => {
         localState,
         [],
         [
+          { type: 'stopJobsPolling' },
           { type: 'requestRepos' },
           {
             type: 'receiveReposSuccess',
@@ -128,7 +129,7 @@ describe('import_projects store actions', () => {
       );
     });
 
-    it('dispatches requestRepos and receiveReposSuccess actions on an unsuccessful request', done => {
+    it('dispatches stopJobsPolling, requestRepos and receiveReposError actions on an unsuccessful request', done => {
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(500);
 
       testAction(
@@ -136,10 +137,39 @@ describe('import_projects store actions', () => {
         null,
         localState,
         [],
-        [{ type: 'requestRepos' }, { type: 'receiveReposError' }],
+        [{ type: 'stopJobsPolling' }, { type: 'requestRepos' }, { type: 'receiveReposError' }],
         done,
       );
     });
+
+    describe('when filtered', () => {
+      beforeEach(() => {
+        localState.filter = 'filter';
+      });
+
+      it('fetches repos with filter applied', done => {
+        mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, payload);
+
+        testAction(
+          fetchRepos,
+          null,
+          localState,
+          [],
+          [
+            { type: 'stopJobsPolling' },
+            { type: 'requestRepos' },
+            {
+              type: 'receiveReposSuccess',
+              payload: convertObjectPropsToCamelCase(payload, { deep: true }),
+            },
+            {
+              type: 'fetchJobs',
+            },
+          ],
+          done,
+        );
+      });
+    });
   });
 
   describe('requestImport', () => {
@@ -249,6 +279,7 @@ describe('import_projects store actions', () => {
 
   describe('fetchJobs', () => {
     let mock;
+    const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
 
     beforeEach(() => {
       localState.jobsPath = `${TEST_HOST}/endpoint.json`;
@@ -263,7 +294,6 @@ describe('import_projects store actions', () => {
     afterEach(() => mock.restore());
 
     it('dispatches requestJobs and receiveJobsSuccess actions on a successful request', done => {
-      const updatedProjects = [{ name: 'imported/project' }, { name: 'provider/repo' }];
       mock.onGet(`${TEST_HOST}/endpoint.json`).reply(200, updatedProjects);
 
       testAction(
@@ -280,5 +310,29 @@ describe('import_projects store actions', () => {
         done,
       );
     });
+
+    describe('when filtered', () => {
+      beforeEach(() => {
+        localState.filter = 'filter';
+      });
+
+      it('fetches realtime changes with filter applied', done => {
+        mock.onGet(`${TEST_HOST}/endpoint.json?filter=filter`).reply(200, updatedProjects);
+
+        testAction(
+          fetchJobs,
+          null,
+          localState,
+          [],
+          [
+            {
+              type: 'receiveJobsSuccess',
+              payload: convertObjectPropsToCamelCase(updatedProjects, { deep: true }),
+            },
+          ],
+          done,
+        );
+      });
+    });
   });
 });
diff --git a/spec/support/controllers/githubish_import_controller_shared_examples.rb b/spec/support/controllers/githubish_import_controller_shared_examples.rb
index 718d9857b18..f23812e7149 100644
--- a/spec/support/controllers/githubish_import_controller_shared_examples.rb
+++ b/spec/support/controllers/githubish_import_controller_shared_examples.rb
@@ -139,6 +139,38 @@ shared_examples 'a GitHub-ish import controller: GET status' do
     expect { get :status, format: :json }
       .not_to exceed_all_query_limit(control_count)
   end
+
+  context 'when filtering' do
+    let(:repo_2) { OpenStruct.new(login: 'emacs', full_name: 'asd/emacs', name: 'emacs', owner: { login: 'owner' }) }
+    let(:project) { create(:project, import_type: provider, namespace: user.namespace, import_status: :finished, import_source: 'example/repo') }
+    let(:group) { create(:group) }
+
+    before do
+      group.add_owner(user)
+      stub_client(repos: [repo, repo_2, org_repo], orgs: [org], org_repos: [org_repo])
+    end
+
+    it 'filters list of repositories by name' do
+      get :status, params: { filter: 'emacs' }, format: :json
+
+      expect(response).to have_gitlab_http_status(200)
+      expect(json_response.dig("imported_projects").count).to eq(0)
+      expect(json_response.dig("provider_repos").count).to eq(1)
+      expect(json_response.dig("provider_repos", 0, "id")).to eq(repo_2.id)
+      expect(json_response.dig("namespaces", 0, "id")).to eq(group.id)
+    end
+
+    context 'when user input contains html' do
+      let(:expected_filter) { 'test' }
+      let(:filter) { "<html>#{expected_filter}</html>" }
+
+      it 'sanitizes user input' do
+        get :status, params: { filter: filter }, format: :json
+
+        expect(assigns(:filter)).to eq(expected_filter)
+      end
+    end
+  end
 end
 
 shared_examples 'a GitHub-ish import controller: POST create' do