diff options
362 files changed, 13661 insertions, 2676 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fed5971233d..6a5050b553f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -587,6 +587,16 @@ codequality: artifacts: paths: [codeclimate.json] +qa:internal: + stage: test + variables: + SETUP_DB: "false" + services: [] + script: + - cd qa/ + - bundle install + - bundle exec rspec + coverage: <<: *dedicated-runner <<: *except-docs diff --git a/app/assets/javascripts/clusters.js b/app/assets/javascripts/clusters.js deleted file mode 100644 index c9fef94efea..00000000000 --- a/app/assets/javascripts/clusters.js +++ /dev/null @@ -1,123 +0,0 @@ -/* globals Flash */ -import Visibility from 'visibilityjs'; -import axios from 'axios'; -import setAxiosCsrfToken from './lib/utils/axios_utils'; -import Poll from './lib/utils/poll'; -import { s__ } from './locale'; -import initSettingsPanels from './settings_panels'; -import Flash from './flash'; - -/** - * Cluster page has 2 separate parts: - * Toggle button - * - * - Polling status while creating or scheduled - * -- Update status area with the response result - */ - -class ClusterService { - constructor(options = {}) { - this.options = options; - setAxiosCsrfToken(); - } - fetchData() { - return axios.get(this.options.endpoint); - } -} - -export default class Clusters { - constructor() { - initSettingsPanels(); - - const dataset = document.querySelector('.js-edit-cluster-form').dataset; - - this.state = { - statusPath: dataset.statusPath, - clusterStatus: dataset.clusterStatus, - clusterStatusReason: dataset.clusterStatusReason, - toggleStatus: dataset.toggleStatus, - }; - - this.service = new ClusterService({ endpoint: this.state.statusPath }); - this.toggleButton = document.querySelector('.js-toggle-cluster'); - this.toggleInput = document.querySelector('.js-toggle-input'); - this.errorContainer = document.querySelector('.js-cluster-error'); - this.successContainer = document.querySelector('.js-cluster-success'); - this.creatingContainer = document.querySelector('.js-cluster-creating'); - this.errorReasonContainer = this.errorContainer.querySelector('.js-error-reason'); - - this.toggleButton.addEventListener('click', this.toggle.bind(this)); - - if (this.state.clusterStatus !== 'created') { - this.updateContainer(this.state.clusterStatus, this.state.clusterStatusReason); - } - - if (this.state.statusPath) { - this.initPolling(); - } - } - - toggle() { - this.toggleButton.classList.toggle('checked'); - this.toggleInput.setAttribute('value', this.toggleButton.classList.contains('checked').toString()); - } - - initPolling() { - this.poll = new Poll({ - resource: this.service, - method: 'fetchData', - successCallback: data => this.handleSuccess(data), - errorCallback: () => Clusters.handleError(), - }); - - if (!Visibility.hidden()) { - this.poll.makeRequest(); - } else { - this.service.fetchData() - .then(data => this.handleSuccess(data)) - .catch(() => Clusters.handleError()); - } - - Visibility.change(() => { - if (!Visibility.hidden()) { - this.poll.restart(); - } else { - this.poll.stop(); - } - }); - } - - static handleError() { - Flash(s__('ClusterIntegration|Something went wrong on our end.')); - } - - handleSuccess(data) { - const { status, status_reason } = data.data; - this.updateContainer(status, status_reason); - } - - hideAll() { - this.errorContainer.classList.add('hidden'); - this.successContainer.classList.add('hidden'); - this.creatingContainer.classList.add('hidden'); - } - - updateContainer(status, error) { - this.hideAll(); - switch (status) { - case 'created': - this.successContainer.classList.remove('hidden'); - break; - case 'errored': - this.errorContainer.classList.remove('hidden'); - this.errorReasonContainer.textContent = error; - break; - case 'scheduled': - case 'creating': - this.creatingContainer.classList.remove('hidden'); - break; - default: - this.hideAll(); - } - } -} diff --git a/app/assets/javascripts/clusters/clusters_bundle.js b/app/assets/javascripts/clusters/clusters_bundle.js new file mode 100644 index 00000000000..dc443475952 --- /dev/null +++ b/app/assets/javascripts/clusters/clusters_bundle.js @@ -0,0 +1,221 @@ +import Visibility from 'visibilityjs'; +import Vue from 'vue'; +import { s__, sprintf } from '../locale'; +import Flash from '../flash'; +import Poll from '../lib/utils/poll'; +import initSettingsPanels from '../settings_panels'; +import eventHub from './event_hub'; +import { + APPLICATION_INSTALLED, + REQUEST_LOADING, + REQUEST_SUCCESS, + REQUEST_FAILURE, +} from './constants'; +import ClustersService from './services/clusters_service'; +import ClustersStore from './stores/clusters_store'; +import applications from './components/applications.vue'; + +/** + * Cluster page has 2 separate parts: + * Toggle button and applications section + * + * - Polling status while creating or scheduled + * - Update status area with the response result + */ + +export default class Clusters { + constructor() { + const { + statusPath, + installHelmPath, + installIngressPath, + installRunnerPath, + clusterStatus, + clusterStatusReason, + helpPath, + } = document.querySelector('.js-edit-cluster-form').dataset; + + this.store = new ClustersStore(); + this.store.setHelpPath(helpPath); + this.store.updateStatus(clusterStatus); + this.store.updateStatusReason(clusterStatusReason); + this.service = new ClustersService({ + endpoint: statusPath, + installHelmEndpoint: installHelmPath, + installIngressEndpoint: installIngressPath, + installRunnerEndpoint: installRunnerPath, + }); + + this.toggle = this.toggle.bind(this); + this.installApplication = this.installApplication.bind(this); + + this.toggleButton = document.querySelector('.js-toggle-cluster'); + this.toggleInput = document.querySelector('.js-toggle-input'); + this.errorContainer = document.querySelector('.js-cluster-error'); + this.successContainer = document.querySelector('.js-cluster-success'); + this.creatingContainer = document.querySelector('.js-cluster-creating'); + this.errorReasonContainer = this.errorContainer.querySelector('.js-error-reason'); + this.successApplicationContainer = document.querySelector('.js-cluster-application-notice'); + + initSettingsPanels(); + this.initApplications(); + + if (this.store.state.status !== 'created') { + this.updateContainer(null, this.store.state.status, this.store.state.statusReason); + } + + this.addListeners(); + if (statusPath) { + this.initPolling(); + } + } + + initApplications() { + const store = this.store; + const el = document.querySelector('#js-cluster-applications'); + + this.applications = new Vue({ + el, + components: { + applications, + }, + data() { + return { + state: store.state, + }; + }, + render(createElement) { + return createElement('applications', { + props: { + applications: this.state.applications, + helpPath: this.state.helpPath, + }, + }); + }, + }); + } + + addListeners() { + this.toggleButton.addEventListener('click', this.toggle); + eventHub.$on('installApplication', this.installApplication); + } + + removeListeners() { + this.toggleButton.removeEventListener('click', this.toggle); + eventHub.$off('installApplication', this.installApplication); + } + + initPolling() { + this.poll = new Poll({ + resource: this.service, + method: 'fetchData', + successCallback: data => this.handleSuccess(data), + errorCallback: () => Clusters.handleError(), + }); + + if (!Visibility.hidden()) { + this.poll.makeRequest(); + } else { + this.service.fetchData() + .then(data => this.handleSuccess(data)) + .catch(() => Clusters.handleError()); + } + + Visibility.change(() => { + if (!Visibility.hidden() && !this.destroyed) { + this.poll.restart(); + } else { + this.poll.stop(); + } + }); + } + + static handleError() { + Flash(s__('ClusterIntegration|Something went wrong on our end.')); + } + + handleSuccess(data) { + const prevStatus = this.store.state.status; + const prevApplicationMap = Object.assign({}, this.store.state.applications); + + this.store.updateStateFromServer(data.data); + + this.checkForNewInstalls(prevApplicationMap, this.store.state.applications); + this.updateContainer(prevStatus, this.store.state.status, this.store.state.statusReason); + } + + toggle() { + this.toggleButton.classList.toggle('checked'); + this.toggleInput.setAttribute('value', this.toggleButton.classList.contains('checked').toString()); + } + + hideAll() { + this.errorContainer.classList.add('hidden'); + this.successContainer.classList.add('hidden'); + this.creatingContainer.classList.add('hidden'); + } + + checkForNewInstalls(prevApplicationMap, newApplicationMap) { + const appTitles = Object.keys(newApplicationMap) + .filter(appId => newApplicationMap[appId].status === APPLICATION_INSTALLED && + prevApplicationMap[appId].status !== APPLICATION_INSTALLED && + prevApplicationMap[appId].status !== null) + .map(appId => newApplicationMap[appId].title); + + if (appTitles.length > 0) { + const text = sprintf(s__('ClusterIntegration|%{appList} was successfully installed on your cluster'), { + appList: appTitles.join(', '), + }); + Flash(text, 'notice', this.successApplicationContainer); + } + } + + updateContainer(prevStatus, status, error) { + this.hideAll(); + + // We poll all the time but only want the `created` banner to show when newly created + if (this.store.state.status !== 'created' || prevStatus !== this.store.state.status) { + switch (status) { + case 'created': + this.successContainer.classList.remove('hidden'); + break; + case 'errored': + this.errorContainer.classList.remove('hidden'); + this.errorReasonContainer.textContent = error; + break; + case 'scheduled': + case 'creating': + this.creatingContainer.classList.remove('hidden'); + break; + default: + this.hideAll(); + } + } + } + + installApplication(appId) { + this.store.updateAppProperty(appId, 'requestStatus', REQUEST_LOADING); + this.store.updateAppProperty(appId, 'requestReason', null); + + this.service.installApplication(appId) + .then(() => { + this.store.updateAppProperty(appId, 'requestStatus', REQUEST_SUCCESS); + }) + .catch(() => { + this.store.updateAppProperty(appId, 'requestStatus', REQUEST_FAILURE); + this.store.updateAppProperty(appId, 'requestReason', s__('ClusterIntegration|Request to begin installing failed')); + }); + } + + destroy() { + this.destroyed = true; + + this.removeListeners(); + + if (this.poll) { + this.poll.stop(); + } + + this.applications.$destroy(); + } +} diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue new file mode 100644 index 00000000000..872abf03ef1 --- /dev/null +++ b/app/assets/javascripts/clusters/components/application_row.vue @@ -0,0 +1,185 @@ +<script> +import { s__, sprintf } from '../../locale'; +import eventHub from '../event_hub'; +import loadingButton from '../../vue_shared/components/loading_button.vue'; +import { + APPLICATION_NOT_INSTALLABLE, + APPLICATION_SCHEDULED, + APPLICATION_INSTALLABLE, + APPLICATION_INSTALLING, + APPLICATION_INSTALLED, + APPLICATION_ERROR, + REQUEST_LOADING, + REQUEST_SUCCESS, + REQUEST_FAILURE, +} from '../constants'; + +export default { + props: { + id: { + type: String, + required: true, + }, + title: { + type: String, + required: true, + }, + titleLink: { + type: String, + required: false, + }, + description: { + type: String, + required: true, + }, + status: { + type: String, + required: false, + }, + statusReason: { + type: String, + required: false, + }, + requestStatus: { + type: String, + required: false, + }, + requestReason: { + type: String, + required: false, + }, + }, + components: { + loadingButton, + }, + computed: { + rowJsClass() { + return `js-cluster-application-row-${this.id}`; + }, + installButtonLoading() { + return !this.status || + this.status === APPLICATION_SCHEDULED || + this.status === APPLICATION_INSTALLING || + this.requestStatus === REQUEST_LOADING; + }, + installButtonDisabled() { + // Avoid the potential for the real-time data to say APPLICATION_INSTALLABLE but + // we already made a request to install and are just waiting for the real-time + // to sync up. + return (this.status !== APPLICATION_INSTALLABLE && this.status !== APPLICATION_ERROR) || + this.requestStatus === REQUEST_LOADING || + this.requestStatus === REQUEST_SUCCESS; + }, + installButtonLabel() { + let label; + if ( + this.status === APPLICATION_NOT_INSTALLABLE || + this.status === APPLICATION_INSTALLABLE || + this.status === APPLICATION_ERROR + ) { + label = s__('ClusterIntegration|Install'); + } else if (this.status === APPLICATION_SCHEDULED || this.status === APPLICATION_INSTALLING) { + label = s__('ClusterIntegration|Installing'); + } else if (this.status === APPLICATION_INSTALLED) { + label = s__('ClusterIntegration|Installed'); + } + + return label; + }, + hasError() { + return this.status === APPLICATION_ERROR || this.requestStatus === REQUEST_FAILURE; + }, + generalErrorDescription() { + return sprintf( + s__('ClusterIntegration|Something went wrong while installing %{title}'), { + title: this.title, + }, + ); + }, + }, + methods: { + installClicked() { + eventHub.$emit('installApplication', this.id); + }, + }, +}; +</script> + +<template> + <div + class="gl-responsive-table-row gl-responsive-table-row-col-span" + :class="rowJsClass" + > + <div + class="gl-responsive-table-row-layout" + role="row" + > + <a + v-if="titleLink" + :href="titleLink" + target="blank" + rel="noopener noreferrer" + role="gridcell" + class="table-section section-15 section-align-top js-cluster-application-title" + > + {{ title }} + </a> + <span + v-else + class="table-section section-15 section-align-top js-cluster-application-title" + > + {{ title }} + </span> + <div + class="table-section section-wrap" + role="gridcell" + > + <div v-html="description"></div> + </div> + <div + class="table-section table-button-footer section-15 section-align-top" + role="gridcell" + > + <div class="btn-group table-action-buttons"> + <loading-button + class="js-cluster-application-install-button" + :loading="installButtonLoading" + :disabled="installButtonDisabled" + :label="installButtonLabel" + @click="installClicked" + /> + </div> + </div> + </div> + <div + v-if="hasError" + class="gl-responsive-table-row-layout" + role="row" + > + <div + class="alert alert-danger alert-block append-bottom-0 table-section section-100" + role="gridcell" + > + <div> + <p class="js-cluster-application-general-error-message"> + {{ generalErrorDescription }} + </p> + <ul v-if="statusReason || requestReason"> + <li + v-if="statusReason" + class="js-cluster-application-status-error-message" + > + {{ statusReason }} + </li> + <li + v-if="requestReason" + class="js-cluster-application-request-error-message" + > + {{ requestReason }} + </li> + </ul> + </div> + </div> + </div> + </div> +</template> diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue new file mode 100644 index 00000000000..e5ae439d26e --- /dev/null +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -0,0 +1,114 @@ +<script> +import _ from 'underscore'; +import { s__, sprintf } from '../../locale'; +import applicationRow from './application_row.vue'; + +export default { + props: { + applications: { + type: Object, + required: false, + default: () => ({}), + }, + helpPath: { + type: String, + required: false, + }, + }, + components: { + applicationRow, + }, + computed: { + generalApplicationDescription() { + return sprintf( + _.escape(s__('ClusterIntegration|Install applications on your cluster. Read more about %{helpLink}')), { + helpLink: `<a href="${this.helpPath}"> + ${_.escape(s__('ClusterIntegration|installing applications'))} + </a>`, + }, + false, + ); + }, + helmTillerDescription() { + return _.escape(s__( + `ClusterIntegration|Helm streamlines installing and managing Kubernets applications. + Tiller runs inside of your Kubernetes Cluster, and manages + releases of your charts.`, + )); + }, + ingressDescription() { + const descriptionParagraph = _.escape(s__( + `ClusterIntegration|Ingress gives you a way to route requests to services based on the + request host or path, centralizing a number of services into a single entrypoint.`, + )); + + const extraCostParagraph = sprintf( + _.escape(s__('ClusterIntegration|%{boldNotice} This will add some extra resources like a load balancer, which incur additional costs. See %{pricingLink}')), { + boldNotice: `<strong>${_.escape(s__('ClusterIntegration|Note:'))}</strong>`, + pricingLink: `<a href="https://cloud.google.com/compute/pricing#lb" target="_blank" rel="noopener noreferrer"> + ${_.escape(s__('ClusterIntegration|GKE pricing'))} + </a>`, + }, + false, + ); + + return ` + <p> + ${descriptionParagraph} + </p> + <p class="append-bottom-0"> + ${extraCostParagraph} + </p> + `; + }, + gitlabRunnerDescription() { + return _.escape(s__( + `ClusterIntegration|GitLab Runner is the open source project that is used to run your jobs + and send the results back to GitLab.`, + )); + }, + }, +}; +</script> + +<template> + <section class="settings no-animate expanded"> + <div class="settings-header"> + <h4> + {{ s__('ClusterIntegration|Applications') }} + </h4> + <p + class="append-bottom-0" + v-html="generalApplicationDescription" + > + </p> + </div> + + <div class="settings-content"> + <div class="append-bottom-20"> + <application-row + id="helm" + :title="applications.helm.title" + title-link="https://docs.helm.sh/" + :description="helmTillerDescription" + :status="applications.helm.status" + :status-reason="applications.helm.statusReason" + :request-status="applications.helm.requestStatus" + :request-reason="applications.helm.requestReason" + /> + <application-row + id="ingress" + :title="applications.ingress.title" + title-link="https://kubernetes.io/docs/concepts/services-networking/ingress/" + :description="ingressDescription" + :status="applications.ingress.status" + :status-reason="applications.ingress.statusReason" + :request-status="applications.ingress.requestStatus" + :request-reason="applications.ingress.requestReason" + /> + <!-- NOTE: Don't forget to update `clusters.scss` min-height for this block and uncomment `application_spec` tests --> + <!-- Add GitLab Runner row, all other plumbing is complete --> + </div> + </div> + </section> +</template> diff --git a/app/assets/javascripts/clusters/constants.js b/app/assets/javascripts/clusters/constants.js new file mode 100644 index 00000000000..93223aefff8 --- /dev/null +++ b/app/assets/javascripts/clusters/constants.js @@ -0,0 +1,12 @@ +// These need to match what is returned from the server +export const APPLICATION_NOT_INSTALLABLE = 'not_installable'; +export const APPLICATION_INSTALLABLE = 'installable'; +export const APPLICATION_SCHEDULED = 'scheduled'; +export const APPLICATION_INSTALLING = 'installing'; +export const APPLICATION_INSTALLED = 'installed'; +export const APPLICATION_ERROR = 'errored'; + +// These are only used client-side +export const REQUEST_LOADING = 'request-loading'; +export const REQUEST_SUCCESS = 'request-success'; +export const REQUEST_FAILURE = 'request-failure'; diff --git a/app/assets/javascripts/clusters/event_hub.js b/app/assets/javascripts/clusters/event_hub.js new file mode 100644 index 00000000000..0948c2e5352 --- /dev/null +++ b/app/assets/javascripts/clusters/event_hub.js @@ -0,0 +1,3 @@ +import Vue from 'vue'; + +export default new Vue(); diff --git a/app/assets/javascripts/clusters/services/clusters_service.js b/app/assets/javascripts/clusters/services/clusters_service.js new file mode 100644 index 00000000000..0ac8e68187d --- /dev/null +++ b/app/assets/javascripts/clusters/services/clusters_service.js @@ -0,0 +1,24 @@ +import axios from 'axios'; +import setAxiosCsrfToken from '../../lib/utils/axios_utils'; + +export default class ClusterService { + constructor(options = {}) { + setAxiosCsrfToken(); + + this.options = options; + this.appInstallEndpointMap = { + helm: this.options.installHelmEndpoint, + ingress: this.options.installIngressEndpoint, + runner: this.options.installRunnerEndpoint, + }; + } + + fetchData() { + return axios.get(this.options.endpoint); + } + + installApplication(appId) { + const endpoint = this.appInstallEndpointMap[appId]; + return axios.post(endpoint); + } +} diff --git a/app/assets/javascripts/clusters/stores/clusters_store.js b/app/assets/javascripts/clusters/stores/clusters_store.js new file mode 100644 index 00000000000..e731cdc3042 --- /dev/null +++ b/app/assets/javascripts/clusters/stores/clusters_store.js @@ -0,0 +1,68 @@ +import { s__ } from '../../locale'; + +export default class ClusterStore { + constructor() { + this.state = { + helpPath: null, + status: null, + statusReason: null, + applications: { + helm: { + title: s__('ClusterIntegration|Helm Tiller'), + status: null, + statusReason: null, + requestStatus: null, + requestReason: null, + }, + ingress: { + title: s__('ClusterIntegration|Ingress'), + status: null, + statusReason: null, + requestStatus: null, + requestReason: null, + }, + runner: { + title: s__('ClusterIntegration|GitLab Runner'), + status: null, + statusReason: null, + requestStatus: null, + requestReason: null, + }, + }, + }; + } + + setHelpPath(helpPath) { + this.state.helpPath = helpPath; + } + + updateStatus(status) { + this.state.status = status; + } + + updateStatusReason(reason) { + this.state.statusReason = reason; + } + + updateAppProperty(appId, prop, value) { + this.state.applications[appId][prop] = value; + } + + updateStateFromServer(serverState = {}) { + this.state.status = serverState.status; + this.state.statusReason = serverState.status_reason; + serverState.applications.forEach((serverAppEntry) => { + const { + name: appId, + status, + status_reason: statusReason, + } = serverAppEntry; + + this.state.applications[appId] = { + ...(this.state.applications[appId] || {}), + status, + statusReason, + }; + }); + } +} diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index 760fb0cdf67..44606989395 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -1,4 +1,5 @@ /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */ +import { s__ } from './locale'; /* global ProjectSelect */ import IssuableIndex from './issuable_index'; /* global Milestone */ @@ -32,6 +33,7 @@ import Labels from './labels'; import LabelManager from './label_manager'; /* global Sidebar */ +import Flash from './flash'; import CommitsList from './commits'; import Issue from './issue'; import BindInOut from './behaviors/bind_in_out'; @@ -543,9 +545,12 @@ import Diff from './diff'; new DueDateSelectors(); break; case 'projects:clusters:show': - import(/* webpackChunkName: "clusters" */ './clusters') + import(/* webpackChunkName: "clusters" */ './clusters/clusters_bundle') .then(cluster => new cluster.default()) // eslint-disable-line new-cap - .catch(() => {}); + .catch((err) => { + Flash(s__('ClusterIntegration|Problem setting up the cluster JavaScript')); + throw err; + }); break; } switch (path[0]) { diff --git a/app/assets/javascripts/repo/components/repo_file.vue b/app/assets/javascripts/repo/components/repo_file.vue index 9f16e6bb2cc..5be47d568e7 100644 --- a/app/assets/javascripts/repo/components/repo_file.vue +++ b/app/assets/javascripts/repo/components/repo_file.vue @@ -54,7 +54,10 @@ <tr class="file" @click.prevent="clickedTreeRow(file)"> - <td :colspan="submoduleColSpan"> + <td + class="multi-file-table-col-name" + :colspan="submoduleColSpan" + > <i class="fa fa-fw file-icon" :class="fileIcon" diff --git a/app/assets/javascripts/repo/components/repo_loading_file.vue b/app/assets/javascripts/repo/components/repo_loading_file.vue index 14c2ddfb615..8fa637d771f 100644 --- a/app/assets/javascripts/repo/components/repo_loading_file.vue +++ b/app/assets/javascripts/repo/components/repo_loading_file.vue @@ -19,7 +19,7 @@ class="loading-file" aria-label="Loading files" > - <td> + <td class="multi-file-table-col-name"> <skeleton-loading-container :small="true" /> diff --git a/app/assets/javascripts/repo/components/repo_sidebar.vue b/app/assets/javascripts/repo/components/repo_sidebar.vue index 3f61d048288..9365b09326f 100644 --- a/app/assets/javascripts/repo/components/repo_sidebar.vue +++ b/app/assets/javascripts/repo/components/repo_sidebar.vue @@ -57,7 +57,7 @@ export default { </strong> </th> <template v-else> - <th class="name"> + <th class="name multi-file-table-col-name"> Name </th> <th class="hidden-sm hidden-xs last-commit"> diff --git a/app/assets/javascripts/smart_interval.js b/app/assets/javascripts/smart_interval.js index 2bf7a3a5d61..8e931995fc6 100644 --- a/app/assets/javascripts/smart_interval.js +++ b/app/assets/javascripts/smart_interval.js @@ -3,9 +3,10 @@ * and controllable by a public API. */ -class SmartInterval { +export default class SmartInterval { /** - * @param { function } opts.callback Function to be called on each iteration (required) + * @param { function } opts.callback Function that returns a promise, called on each iteration + * unless still in progress (required) * @param { milliseconds } opts.startingInterval `currentInterval` is set to this initially * @param { milliseconds } opts.maxInterval `currentInterval` will be incremented to this * @param { milliseconds } opts.hiddenInterval `currentInterval` is set to this @@ -42,13 +43,16 @@ class SmartInterval { const cfg = this.cfg; const state = this.state; - if (cfg.immediateExecution) { + if (cfg.immediateExecution && !this.isLoading) { cfg.immediateExecution = false; - cfg.callback(); + this.triggerCallback(); } state.intervalId = window.setInterval(() => { - cfg.callback(); + if (this.isLoading) { + return; + } + this.triggerCallback(); if (this.getCurrentInterval() === cfg.maxInterval) { return; @@ -76,7 +80,7 @@ class SmartInterval { // start a timer, using the existing interval resume() { - this.stopTimer(); // stop exsiting timer, in case timer was not previously stopped + this.stopTimer(); // stop existing timer, in case timer was not previously stopped this.start(); } @@ -104,6 +108,18 @@ class SmartInterval { this.initPageUnloadHandling(); } + triggerCallback() { + this.isLoading = true; + this.cfg.callback() + .then(() => { + this.isLoading = false; + }) + .catch((err) => { + this.isLoading = false; + throw err; + }); + } + initVisibilityChangeHandling() { // cancel interval when tab no longer shown (prevents cached pages from polling) document.addEventListener('visibilitychange', this.handleVisibilityChange.bind(this)); @@ -154,4 +170,3 @@ class SmartInterval { } } -window.gl.SmartInterval = SmartInterval; diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.js b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.js deleted file mode 100644 index 029832bdd27..00000000000 --- a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.js +++ /dev/null @@ -1,90 +0,0 @@ -import PipelineStage from '../../pipelines/components/stage.vue'; -import ciIcon from '../../vue_shared/components/ci_icon.vue'; -import icon from '../../vue_shared/components/icon.vue'; - -export default { - name: 'MRWidgetPipeline', - props: { - mr: { type: Object, required: true }, - }, - components: { - 'pipeline-stage': PipelineStage, - ciIcon, - icon, - }, - computed: { - hasPipeline() { - return this.mr.pipeline && Object.keys(this.mr.pipeline).length > 0; - }, - hasCIError() { - const { hasCI, ciStatus } = this.mr; - - return hasCI && !ciStatus; - }, - stageText() { - return this.mr.pipeline.details.stages.length > 1 ? 'stages' : 'stage'; - }, - status() { - return this.mr.pipeline.details.status || {}; - }, - }, - template: ` - <div - v-if="hasPipeline || hasCIError" - class="mr-widget-heading"> - <div class="ci-widget media"> - <template v-if="hasCIError"> - <div class="ci-status-icon ci-status-icon-failed ci-error js-ci-error append-right-10"> - <span - aria-hidden="true"> - <icon - name="status_failed"/> - </span> - </div> - <div class="media-body"> - Could not connect to the CI server. Please check your settings and try again - </div> - </template> - <template v-else-if="hasPipeline"> - <div class="ci-status-icon append-right-10"> - <a - class="icon-link" - :href="this.status.details_path"> - <ci-icon :status="status" /> - </a> - </div> - <div class="media-body"> - <span> - Pipeline - <a - :href="mr.pipeline.path" - class="pipeline-id">#{{mr.pipeline.id}}</a> - </span> - <span class="mr-widget-pipeline-graph"> - <span class="stage-cell"> - <div - v-if="mr.pipeline.details.stages.length > 0" - v-for="stage in mr.pipeline.details.stages" - class="stage-container dropdown js-mini-pipeline-graph"> - <pipeline-stage :stage="stage" /> - </div> - </span> - </span> - <span> - {{mr.pipeline.details.status.label}} for - <a - :href="mr.pipeline.commit.commit_path" - class="commit-sha js-commit-link"> - {{mr.pipeline.commit.short_id}}</a>. - </span> - <span - v-if="mr.pipeline.coverage" - class="js-mr-coverage"> - Coverage {{mr.pipeline.coverage}}% - </span> - </div> - </template> - </div> - </div> - `, -}; diff --git a/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue new file mode 100644 index 00000000000..dbc65462377 --- /dev/null +++ b/app/assets/javascripts/vue_merge_request_widget/components/mr_widget_pipeline.vue @@ -0,0 +1,104 @@ +<script> + import pipelineStage from '../../pipelines/components/stage.vue'; + import ciIcon from '../../vue_shared/components/ci_icon.vue'; + import icon from '../../vue_shared/components/icon.vue'; + + export default { + name: 'MRWidgetPipeline', + props: { + pipeline: { + type: Object, + required: true, + }, + // This prop needs to be camelCase, html attributes are case insensive + // https://vuejs.org/v2/guide/components.html#camelCase-vs-kebab-case + hasCi: { + type: Boolean, + required: false, + }, + ciStatus: { + type: String, + required: false, + }, + }, + components: { + pipelineStage, + ciIcon, + icon, + }, + computed: { + hasPipeline() { + return this.pipeline && Object.keys(this.pipeline).length > 0; + }, + hasCIError() { + return this.hasCi && !this.ciStatus; + }, + status() { + return this.pipeline.details && + this.pipeline.details.status ? this.pipeline.details.status : {}; + }, + hasStages() { + return this.pipeline.details && + this.pipeline.details.stages && + this.pipeline.details.stages.length; + }, + }, + }; +</script> + +<template> + <div + v-if="hasPipeline || hasCIError" + class="mr-widget-heading"> + <div class="ci-widget media"> + <template v-if="hasCIError"> + <div class="ci-status-icon ci-status-icon-failed ci-error js-ci-error append-right-10"> + <icon name="status_failed" /> + </div> + <div class="media-body"> + Could not connect to the CI server. Please check your settings and try again + </div> + </template> + <template v-else-if="hasPipeline"> + <a + class="append-right-10" + :href="this.status.details_path"> + <ci-icon :status="status" /> + </a> + + <div class="media-body"> + Pipeline + <a + :href="pipeline.path" + class="pipeline-id"> + #{{pipeline.id}} + </a> + + {{pipeline.details.status.label}} for + + <a + :href="pipeline.commit.commit_path" + class="commit-sha js-commit-link"> + {{pipeline.commit.short_id}}</a>. + + <span class="mr-widget-pipeline-graph"> + <span class="stage-cell"> + <div + v-if="hasStages" + v-for="(stage, i) in pipeline.details.stages" + :key="i" + class="stage-container dropdown js-mini-pipeline-graph"> + <pipeline-stage :stage="stage" /> + </div> + </span> + </span> + + <template v-if="pipeline.coverage"> + Coverage {{pipeline.coverage}}% + </template> + + </div> + </template> + </div> + </div> +</template> diff --git a/app/assets/javascripts/vue_merge_request_widget/dependencies.js b/app/assets/javascripts/vue_merge_request_widget/dependencies.js index 49340c232c8..5bd8b99420a 100644 --- a/app/assets/javascripts/vue_merge_request_widget/dependencies.js +++ b/app/assets/javascripts/vue_merge_request_widget/dependencies.js @@ -13,7 +13,7 @@ export { default as Vue } from 'vue'; export { default as SmartInterval } from '~/smart_interval'; export { default as WidgetHeader } from './components/mr_widget_header'; export { default as WidgetMergeHelp } from './components/mr_widget_merge_help'; -export { default as WidgetPipeline } from './components/mr_widget_pipeline'; +export { default as WidgetPipeline } from './components/mr_widget_pipeline.vue'; export { default as WidgetDeployment } from './components/mr_widget_deployment'; export { default as WidgetRelatedLinks } from './components/mr_widget_related_links'; export { default as MergedState } from './components/states/mr_widget_merged'; diff --git a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js index 4f497b204a3..f82938aa8a9 100644 --- a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js +++ b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js @@ -1,3 +1,4 @@ +import SmartInterval from '~/smart_interval'; import Flash from '../flash'; import { WidgetHeader, @@ -81,7 +82,7 @@ export default { return new MRWidgetService(endpoints); }, checkStatus(cb) { - this.service.checkStatus() + return this.service.checkStatus() .then(res => res.json()) .then((res) => { this.handleNotification(res); @@ -97,7 +98,7 @@ export default { }); }, initPolling() { - this.pollingInterval = new gl.SmartInterval({ + this.pollingInterval = new SmartInterval({ callback: this.checkStatus, startingInterval: 10000, maxInterval: 30000, @@ -106,7 +107,7 @@ export default { }); }, initDeploymentsPolling() { - this.deploymentsInterval = new gl.SmartInterval({ + this.deploymentsInterval = new SmartInterval({ callback: this.fetchDeployments, startingInterval: 30000, maxInterval: 120000, @@ -121,7 +122,7 @@ export default { } }, fetchDeployments() { - this.service.fetchDeployments() + return this.service.fetchDeployments() .then(res => res.json()) .then((res) => { if (res.length) { @@ -235,7 +236,10 @@ export default { <mr-widget-header :mr="mr" /> <mr-widget-pipeline v-if="shouldRenderPipelines" - :mr="mr" /> + :pipeline="mr.pipeline" + :ci-status="mr.ciStatus" + :has-ci="mr.hasCI" + /> <mr-widget-deployment v-if="shouldRenderDeployments" :mr="mr" diff --git a/app/assets/javascripts/vue_shared/components/loading_button.vue b/app/assets/javascripts/vue_shared/components/loading_button.vue index 6670b554faf..0cc2653761c 100644 --- a/app/assets/javascripts/vue_shared/components/loading_button.vue +++ b/app/assets/javascripts/vue_shared/components/loading_button.vue @@ -26,6 +26,11 @@ export default { required: false, default: false, }, + disabled: { + type: Boolean, + required: false, + default: false, + }, label: { type: String, required: false, @@ -47,7 +52,7 @@ export default { class="btn btn-align-content" @click="onClick" type="button" - :disabled="loading" + :disabled="loading || disabled" > <transition name="fade"> <loading-icon diff --git a/app/assets/stylesheets/framework/animations.scss b/app/assets/stylesheets/framework/animations.scss index 1b944831082..374988bb590 100644 --- a/app/assets/stylesheets/framework/animations.scss +++ b/app/assets/stylesheets/framework/animations.scss @@ -23,16 +23,6 @@ @include webkit-prefix(animation-duration, 2s); } - &.spin-cw { - transform-origin: center; - animation: spin 4s linear infinite; - } - - &.spin-ccw { - transform-origin: center; - animation: spin 4s linear infinite reverse; - } - &.flipOutX, &.flipOutY, &.bounceIn, @@ -281,9 +271,3 @@ a { transform: translateX(468px); } } - -@keyframes spin { - 100% { - transform: rotate(360deg); - } -} diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index c4a95afc4d2..b2f26cf7159 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -294,6 +294,7 @@ .btn-align-content { display: flex; + justify-content: center; align-items: center; } diff --git a/app/assets/stylesheets/framework/wells.scss b/app/assets/stylesheets/framework/wells.scss index 5f9756bf58a..68824ff8418 100644 --- a/app/assets/stylesheets/framework/wells.scss +++ b/app/assets/stylesheets/framework/wells.scss @@ -52,6 +52,37 @@ .label.label-gray { background-color: $well-expand-item; } + + .branches { + display: inline; + } + + .branch-link { + margin-bottom: 2px; + } + + .limit-box { + cursor: pointer; + display: inline-flex; + align-items: center; + background-color: $red-100; + border-radius: $border-radius-default; + text-align: center; + + &:hover { + background-color: $red-200; + } + + .limit-icon { + margin: 0 8px; + } + + .limit-message { + line-height: 16px; + margin-right: 8px; + font-size: 12px; + } + } } .light-well { diff --git a/app/assets/stylesheets/pages/clusters.scss b/app/assets/stylesheets/pages/clusters.scss index 5c91579c69c..e5b9e1f2de6 100644 --- a/app/assets/stylesheets/pages/clusters.scss +++ b/app/assets/stylesheets/pages/clusters.scss @@ -3,3 +3,8 @@ background-color: $white-light; } } + +.cluster-applications-table { + // Wait for the Vue to kick-in and render the applications block + min-height: 302px; +} diff --git a/app/assets/stylesheets/pages/repo.scss b/app/assets/stylesheets/pages/repo.scss index fee4638e20f..d93c51d5448 100644 --- a/app/assets/stylesheets/pages/repo.scss +++ b/app/assets/stylesheets/pages/repo.scss @@ -298,3 +298,7 @@ width: 100%; } } + +.multi-file-table-col-name { + width: 350px; +} diff --git a/app/controllers/concerns/issuable_actions.rb b/app/controllers/concerns/issuable_actions.rb index 9c222549cdc..072dffaff7a 100644 --- a/app/controllers/concerns/issuable_actions.rb +++ b/app/controllers/concerns/issuable_actions.rb @@ -9,9 +9,7 @@ module IssuableActions def show respond_to do |format| - format.html do - render show_view - end + format.html format.json do render json: serializer.represent(issuable, serializer: params[:serializer]) end @@ -152,10 +150,6 @@ module IssuableActions end end - def show_view - 'show' - end - def serializer raise NotImplementedError end diff --git a/app/controllers/concerns/issuable_collections.rb b/app/controllers/concerns/issuable_collections.rb index 3181f517087..2b011bc87b0 100644 --- a/app/controllers/concerns/issuable_collections.rb +++ b/app/controllers/concerns/issuable_collections.rb @@ -4,58 +4,44 @@ module IssuableCollections include Gitlab::IssuableMetadata included do - helper_method :issues_finder - helper_method :merge_requests_finder + helper_method :finder end private - def set_issues_index - @collection_type = "Issue" - @issues = issues_collection - @issues = @issues.page(params[:page]) - @issuable_meta_data = issuable_meta_data(@issues, @collection_type) - @total_pages = issues_page_count(@issues) + def set_issuables_index + @issuables = issuables_collection + @issuables = @issuables.page(params[:page]) + @issuable_meta_data = issuable_meta_data(@issuables, collection_type) + @total_pages = issuable_page_count - return if redirect_out_of_range(@issues, @total_pages) + return if redirect_out_of_range(@total_pages) if params[:label_name].present? - @labels = LabelsFinder.new(current_user, project_id: @project.id, title: params[:label_name]).execute + labels_params = { project_id: @project.id, title: params[:label_name] } + @labels = LabelsFinder.new(current_user, labels_params).execute end @users = [] - end - - def issues_collection - issues_finder.execute.preload(:project, :author, :assignees, :labels, :milestone, project: :namespace) - end - - def merge_requests_collection - merge_requests_finder.execute.preload( - :source_project, - :target_project, - :author, - :assignee, - :labels, - :milestone, - head_pipeline: :project, - target_project: :namespace, - merge_request_diff: :merge_request_diff_commits - ) - end + if params[:assignee_id].present? + assignee = User.find_by_id(params[:assignee_id]) + @users.push(assignee) if assignee + end - def issues_finder - @issues_finder ||= issuable_finder_for(IssuesFinder) + if params[:author_id].present? + author = User.find_by_id(params[:author_id]) + @users.push(author) if author + end end - def merge_requests_finder - @merge_requests_finder ||= issuable_finder_for(MergeRequestsFinder) + def issuables_collection + finder.execute.preload(preload_for_collection) end - def redirect_out_of_range(relation, total_pages) + def redirect_out_of_range(total_pages) return false if total_pages.zero? - out_of_range = relation.current_page > total_pages + out_of_range = @issuables.current_page > total_pages if out_of_range redirect_to(url_for(params.merge(page: total_pages, only_path: true))) @@ -64,12 +50,8 @@ module IssuableCollections out_of_range end - def issues_page_count(relation) - page_count_for_relation(relation, issues_finder.row_count) - end - - def merge_requests_page_count(relation) - page_count_for_relation(relation, merge_requests_finder.row_count) + def issuable_page_count + page_count_for_relation(@issuables, finder.row_count) end def page_count_for_relation(relation, row_count) @@ -145,4 +127,31 @@ module IssuableCollections else value end end + + def finder + return @finder if defined?(@finder) + + @finder = issuable_finder_for(@finder_type) + end + + def collection_type + @collection_type ||= case finder + when IssuesFinder + 'Issue' + when MergeRequestsFinder + 'MergeRequest' + end + end + + def preload_for_collection + @preload_for_collection ||= case collection_type + when 'Issue' + [:project, :author, :assignees, :labels, :milestone, project: :namespace] + when 'MergeRequest' + [ + :source_project, :target_project, :author, :assignee, :labels, :milestone, + head_pipeline: :project, target_project: :namespace, merge_request_diff: :merge_request_diff_commits + ] + end + end end diff --git a/app/controllers/concerns/issues_action.rb b/app/controllers/concerns/issues_action.rb index 404559c8707..ad594903331 100644 --- a/app/controllers/concerns/issues_action.rb +++ b/app/controllers/concerns/issues_action.rb @@ -3,14 +3,14 @@ module IssuesAction include IssuableCollections def issues - @label = issues_finder.labels.first + @finder_type = IssuesFinder + @label = finder.labels.first - @issues = issues_collection + @issues = issuables_collection .non_archived .page(params[:page]) - @collection_type = "Issue" - @issuable_meta_data = issuable_meta_data(@issues, @collection_type) + @issuable_meta_data = issuable_meta_data(@issues, collection_type) respond_to do |format| format.html diff --git a/app/controllers/concerns/merge_requests_action.rb b/app/controllers/concerns/merge_requests_action.rb index d3c8e4888bc..8b569a01afd 100644 --- a/app/controllers/concerns/merge_requests_action.rb +++ b/app/controllers/concerns/merge_requests_action.rb @@ -3,13 +3,12 @@ module MergeRequestsAction include IssuableCollections def merge_requests - @label = merge_requests_finder.labels.first + @finder_type = MergeRequestsFinder + @label = finder.labels.first - @merge_requests = merge_requests_collection - .page(params[:page]) + @merge_requests = issuables_collection.page(params[:page]) - @collection_type = "MergeRequest" - @issuable_meta_data = issuable_meta_data(@merge_requests, @collection_type) + @issuable_meta_data = issuable_meta_data(@merge_requests, collection_type) end private diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index ab18d86dcae..b8ba7921613 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -43,7 +43,7 @@ class Import::GithubController < Import::BaseController @target_namespace = find_or_create_namespace(namespace_path, current_user.namespace_path) if can?(current_user, :create_projects, @target_namespace) - @project = Gitlab::GithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params, type: provider).execute + @project = Gitlab::LegacyGithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params, type: provider).execute else render 'unauthorized' end @@ -52,7 +52,7 @@ class Import::GithubController < Import::BaseController private def client - @client ||= Gitlab::GithubImport::Client.new(session[access_token_key], client_options) + @client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options) end def verify_import_enabled diff --git a/app/controllers/projects/clusters/applications_controller.rb b/app/controllers/projects/clusters/applications_controller.rb new file mode 100644 index 00000000000..90c7fa62216 --- /dev/null +++ b/app/controllers/projects/clusters/applications_controller.rb @@ -0,0 +1,25 @@ +class Projects::Clusters::ApplicationsController < Projects::ApplicationController + before_action :cluster + before_action :application_class, only: [:create] + before_action :authorize_read_cluster! + before_action :authorize_create_cluster!, only: [:create] + + def create + Clusters::Applications::ScheduleInstallationService.new(project, current_user, + application_class: @application_class, + cluster: @cluster).execute + head :no_content + rescue StandardError + head :bad_request + end + + private + + def cluster + @cluster ||= project.clusters.find(params[:id]) || render_404 + end + + def application_class + @application_class ||= Clusters::Cluster::APPLICATIONS[params[:application]] || render_404 + end +end diff --git a/app/controllers/projects/commit_controller.rb b/app/controllers/projects/commit_controller.rb index a62f05db7db..494d412b532 100644 --- a/app/controllers/projects/commit_controller.rb +++ b/app/controllers/projects/commit_controller.rb @@ -16,6 +16,8 @@ class Projects::CommitController < Projects::ApplicationController before_action :define_note_vars, only: [:show, :diff_for_path] before_action :authorize_edit_tree!, only: [:revert, :cherry_pick] + BRANCH_SEARCH_LIMIT = 1000 + def show apply_diff_view_cookie! @@ -56,8 +58,14 @@ class Projects::CommitController < Projects::ApplicationController end def branches - @branches = @project.repository.branch_names_contains(commit.id) - @tags = @project.repository.tag_names_contains(commit.id) + # branch_names_contains/tag_names_contains can take a long time when there are thousands of + # branches/tags - each `git branch --contains xxx` request can consume a cpu core. + # so only do the query when there are a manageable number of branches/tags + @branches_limit_exceeded = @project.repository.branch_count > BRANCH_SEARCH_LIMIT + @branches = @branches_limit_exceeded ? [] : @project.repository.branch_names_contains(commit.id) + + @tags_limit_exceeded = @project.repository.tag_count > BRANCH_SEARCH_LIMIT + @tags = @tags_limit_exceeded ? [] : @project.repository.tag_names_contains(commit.id) render layout: false end diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index d4e763aa5b8..dbc9106ba6d 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -10,7 +10,7 @@ class Projects::IssuesController < Projects::ApplicationController before_action :check_issues_available! before_action :issue, except: [:index, :new, :create, :bulk_update] - before_action :set_issues_index, only: [:index] + before_action :set_issuables_index, only: [:index] # Allow write(create) issue before_action :authorize_create_issue!, only: [:new, :create] @@ -24,15 +24,7 @@ class Projects::IssuesController < Projects::ApplicationController respond_to :html def index - if params[:assignee_id].present? - assignee = User.find_by_id(params[:assignee_id]) - @users.push(assignee) if assignee - end - - if params[:author_id].present? - author = User.find_by_id(params[:author_id]) - @users.push(author) if author - end + @issues = @issuables respond_to do |format| format.html @@ -252,4 +244,9 @@ class Projects::IssuesController < Projects::ApplicationController update_params = issue_params.merge(spammable_params) Issues::UpdateService.new(project, current_user, update_params) end + + def set_issuables_index + @finder_type = IssuesFinder + super + end end diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index c068f95b55a..22de6680511 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -10,33 +10,12 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort] + before_action :set_issuables_index, only: [:index] + before_action :authenticate_user!, only: [:assign_related_issues] def index - @collection_type = "MergeRequest" - @merge_requests = merge_requests_collection - @merge_requests = @merge_requests.page(params[:page]) - @merge_requests = @merge_requests.preload(merge_request_diff: :merge_request) - @issuable_meta_data = issuable_meta_data(@merge_requests, @collection_type) - @total_pages = merge_requests_page_count(@merge_requests) - - return if redirect_out_of_range(@merge_requests, @total_pages) - - if params[:label_name].present? - labels_params = { project_id: @project.id, title: params[:label_name] } - @labels = LabelsFinder.new(current_user, labels_params).execute - end - - @users = [] - if params[:assignee_id].present? - assignee = User.find_by_id(params[:assignee_id]) - @users.push(assignee) if assignee - end - - if params[:author_id].present? - author = User.find_by_id(params[:author_id]) - @users.push(author) if author - end + @merge_requests = @issuables respond_to do |format| format.html @@ -336,4 +315,9 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo @target_project = @merge_request.target_project @target_branches = @merge_request.target_project.repository.branch_names end + + def set_issuables_index + @finder_type = MergeRequestsFinder + super + end end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 1688121e27e..2a473ec0cec 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -275,7 +275,8 @@ class ProjectsController < Projects::ApplicationController @project_wiki = @project.wiki @wiki_home = @project_wiki.find_page('home', params[:version_id]) elsif @project.feature_available?(:issues, current_user) - @issues = issues_collection.page(params[:page]) + @finder_type = IssuesFinder + @issues = issuables_collection.page(params[:page]) @collection_type = 'Issue' @issuable_meta_data = issuable_meta_data(@issues, @collection_type) end diff --git a/app/helpers/commits_helper.rb b/app/helpers/commits_helper.rb index ef22cafc2e2..f9a666fa1e6 100644 --- a/app/helpers/commits_helper.rb +++ b/app/helpers/commits_helper.rb @@ -60,23 +60,33 @@ module CommitsHelper branches.include?(project.default_branch) ? branches.delete(project.default_branch) : branches.pop end + # Returns a link formatted as a commit branch link + def commit_branch_link(url, text) + link_to(url, class: 'label label-gray ref-name branch-link') do + icon('code-fork') + " #{text}" + end + end + # Returns the sorted alphabetically links to branches, separated by a comma def commit_branches_links(project, branches) branches.sort.map do |branch| - link_to(project_ref_path(project, branch), class: "label label-gray ref-name") do - icon('code-fork') + " #{branch}" - end - end.join(" ").html_safe + commit_branch_link(project_ref_path(project, branch), branch) + end.join(' ').html_safe + end + + # Returns a link formatted as a commit tag link + def commit_tag_link(url, text) + link_to(url, class: 'label label-gray ref-name') do + icon('tag') + " #{text}" + end end # Returns the sorted links to tags, separated by a comma def commit_tags_links(project, tags) sorted = VersionSorter.rsort(tags) sorted.map do |tag| - link_to(project_ref_path(project, tag), class: "label label-gray ref-name") do - icon('tag') + " #{tag}" - end - end.join(" ").html_safe + commit_tag_link(project_ref_path(project, tag), tag) + end.join(' ').html_safe end def link_to_browse_code(project, commit) diff --git a/app/helpers/issuables_helper.rb b/app/helpers/issuables_helper.rb index 85407e38532..a9840d19178 100644 --- a/app/helpers/issuables_helper.rb +++ b/app/helpers/issuables_helper.rb @@ -249,8 +249,6 @@ module IssuablesHelper end def issuables_count_for_state(issuable_type, state) - finder = public_send("#{issuable_type}_finder") # rubocop:disable GitlabSecurity/PublicSend - Gitlab::IssuablesCountForState.new(finder)[state] end diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb new file mode 100644 index 00000000000..c7949d11ef8 --- /dev/null +++ b/app/models/clusters/applications/helm.rb @@ -0,0 +1,35 @@ +module Clusters + module Applications + class Helm < ActiveRecord::Base + self.table_name = 'clusters_applications_helm' + + include ::Clusters::Concerns::ApplicationStatus + + belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id + + default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION + + validates :cluster, presence: true + + after_initialize :set_initial_status + + def self.application_name + self.to_s.demodulize.underscore + end + + def set_initial_status + return unless not_installable? + + self.status = 'installable' if cluster&.platform_kubernetes_active? + end + + def name + self.class.application_name + end + + def install_command + Gitlab::Kubernetes::Helm::InstallCommand.new(name, true) + end + end + end +end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb new file mode 100644 index 00000000000..44bd979741e --- /dev/null +++ b/app/models/clusters/applications/ingress.rb @@ -0,0 +1,44 @@ +module Clusters + module Applications + class Ingress < ActiveRecord::Base + self.table_name = 'clusters_applications_ingress' + + include ::Clusters::Concerns::ApplicationStatus + + belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id + + validates :cluster, presence: true + + default_value_for :ingress_type, :nginx + default_value_for :version, :nginx + + after_initialize :set_initial_status + + enum ingress_type: { + nginx: 1 + } + + def self.application_name + self.to_s.demodulize.underscore + end + + def set_initial_status + return unless not_installable? + + self.status = 'installable' if cluster&.application_helm_installed? + end + + def name + self.class.application_name + end + + def chart + 'stable/nginx-ingress' + end + + def install_command + Gitlab::Kubernetes::Helm::InstallCommand.new(name, false, chart) + end + end + end +end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 955dba51745..185d9473aab 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -4,6 +4,11 @@ module Clusters self.table_name = 'clusters' + APPLICATIONS = { + Applications::Helm.application_name => Applications::Helm, + Applications::Ingress.application_name => Applications::Ingress + }.freeze + belongs_to :user has_many :cluster_projects, class_name: 'Clusters::Project' @@ -15,6 +20,9 @@ module Clusters # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :application_helm, class_name: 'Clusters::Applications::Helm' + has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' + accepts_nested_attributes_for :provider_gcp, update_only: true accepts_nested_attributes_for :platform_kubernetes, update_only: true @@ -28,10 +36,12 @@ module Clusters delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true - delegate :status_name, to: :provider, allow_nil: true delegate :on_creation?, to: :provider, allow_nil: true delegate :update_kubernetes_integration!, to: :platform, allow_nil: true + delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true + delegate :installed?, to: :application_helm, prefix: true, allow_nil: true + enum platform_type: { kubernetes: 1 } @@ -44,6 +54,21 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } + def status_name + if provider + provider.status_name + else + :created + end + end + + def applications + [ + application_helm || build_application_helm, + application_ingress || build_application_ingress + ] + end + def provider return provider_gcp if gcp? end @@ -59,6 +84,10 @@ module Clusters end alias_method :project, :first_project + def kubeclient + platform_kubernetes.kubeclient if kubernetes? + end + private def restrict_modification diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb new file mode 100644 index 00000000000..7b7c8eac773 --- /dev/null +++ b/app/models/clusters/concerns/application_status.rb @@ -0,0 +1,43 @@ +module Clusters + module Concerns + module ApplicationStatus + extend ActiveSupport::Concern + + included do + state_machine :status, initial: :not_installable do + state :not_installable, value: -2 + state :errored, value: -1 + state :installable, value: 0 + state :scheduled, value: 1 + state :installing, value: 2 + state :installed, value: 3 + + event :make_scheduled do + transition [:installable, :errored] => :scheduled + end + + event :make_installing do + transition [:scheduled] => :installing + end + + event :make_installed do + transition [:installing] => :installed + end + + event :make_errored do + transition any => :errored + end + + before_transition any => [:scheduled] do |app_status, _| + app_status.status_reason = nil + end + + before_transition any => [:errored] do |app_status, transition| + status_reason = transition.args.first + app_status.status_reason = status_reason if status_reason + end + end + end + end + end +end diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index b11701797c2..6dc1ee810d3 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -55,6 +55,10 @@ module Clusters self.class.namespace_for_project(project) if project end + def kubeclient + @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service? + end + def update_kubernetes_integration! raise 'Kubernetes service already configured' unless manages_kubernetes_service? @@ -70,6 +74,10 @@ module Clusters ) end + def active? + manages_kubernetes_service? + end + private def enforce_namespace_to_lower_case diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index a928b9d6367..c008fb91a16 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -17,6 +17,8 @@ module Issuable include Importable include Editable include AfterCommitQueue + include Sortable + include CreatedAtFilterable # This object is used to gather issuable meta data for displaying # upvotes, downvotes, notes and closing merge requests count for issues and merge requests diff --git a/app/models/issue.rb b/app/models/issue.rb index fc590f9257e..3b3c7fb7f8b 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -5,11 +5,9 @@ class Issue < ActiveRecord::Base include Issuable include Noteable include Referable - include Sortable include Spammable include FasterCacheKeys include RelativePositioning - include CreatedAtFilterable include TimeTrackable DueDateStruct = Struct.new(:title, :name).freeze diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 919391d2e47..efd8cca2947 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -3,9 +3,7 @@ class MergeRequest < ActiveRecord::Base include Issuable include Noteable include Referable - include Sortable include IgnorableColumn - include CreatedAtFilterable include TimeTrackable ignore_column :locked_at, diff --git a/app/models/project.rb b/app/models/project.rb index 379bab27d17..bae16b6b2af 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -189,6 +189,7 @@ class Project < ActiveRecord::Base has_one :cluster_project, class_name: 'Clusters::Project' has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -364,6 +365,7 @@ class Project < ActiveRecord::Base scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } scope :excluding_project, ->(project) { where.not(id: project) } + scope :import_started, -> { where(import_status: 'started') } state_machine :import_status, initial: :none do event :import_schedule do @@ -1189,6 +1191,10 @@ class Project < ActiveRecord::Base !!repository.exists? end + def wiki_repository_exists? + wiki.repository_exists? + end + # update visibility_level of forks def update_forks_visibility_level return unless visibility_level < visibility_level_was @@ -1432,6 +1438,31 @@ class Project < ActiveRecord::Base reload_repository! end + def after_import + repository.after_import + import_finish + remove_import_jid + update_project_counter_caches + end + + def update_project_counter_caches + classes = [ + Projects::OpenIssuesCountService, + Projects::OpenMergeRequestsCountService + ] + + classes.each do |klass| + klass.new(self).refresh_cache + end + end + + def remove_import_jid + return unless import_jid + + Gitlab::SidekiqStatus.unset(import_jid) + update_column(:import_jid, nil) + end + def running_or_pending_build_count(force: false) Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do builds.running_or_pending.count(:all) @@ -1689,6 +1720,17 @@ class Project < ActiveRecord::Base Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki)) end + # Refreshes the expiration time of the associated import job ID. + # + # This method can be used by asynchronous importers to refresh the status, + # preventing the StuckImportJobsWorker from marking the import as failed. + def refresh_import_jid_expiration + return unless import_jid + + Gitlab::SidekiqStatus + .set(import_jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + end + private def storage diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 5c0b3338a62..5080acffb3c 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -136,6 +136,10 @@ class KubernetesService < DeploymentService { pods: read_pods } end + def kubeclient + @kubeclient ||= build_kubeclient! + end + TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze private diff --git a/app/models/repository.rb b/app/models/repository.rb index eb7766d040c..6bdee538172 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -973,6 +973,10 @@ class Repository raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end + def remote_exists?(name) + raw_repository.remote_exists?(name) + end + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) raw_repository.compare_source_branch(target_branch_name, source_repository.raw_repository, source_branch_name, straight: straight) end diff --git a/app/models/user.rb b/app/models/user.rb index aa88cda4dc0..f436efd604f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -267,18 +267,23 @@ class User < ActiveRecord::Base end end + def for_github_id(id) + joins(:identities) + .where(identities: { provider: :github, extern_uid: id.to_s }) + end + # Find a User by their primary email or any associated secondary email def find_by_any_email(email) - sql = 'SELECT * - FROM users - WHERE id IN ( - SELECT id FROM users WHERE email = :email - UNION - SELECT emails.user_id FROM emails WHERE email = :email - ) - LIMIT 1;' + by_any_email(email).take + end + + # Returns a relation containing all the users for the given Email address + def by_any_email(email) + users = where(email: email) + emails = joins(:emails).where(emails: { email: email }) + union = Gitlab::SQL::Union.new([users, emails]) - User.find_by_sql([sql, { email: email }]).first + from("(#{union.to_sql}) #{table_name}") end def filter(filter_name) diff --git a/app/serializers/cluster_application_entity.rb b/app/serializers/cluster_application_entity.rb new file mode 100644 index 00000000000..3f9a275ad08 --- /dev/null +++ b/app/serializers/cluster_application_entity.rb @@ -0,0 +1,5 @@ +class ClusterApplicationEntity < Grape::Entity + expose :name + expose :status_name, as: :status + expose :status_reason +end diff --git a/app/serializers/cluster_entity.rb b/app/serializers/cluster_entity.rb index 08a113c4d8a..7e5b0997878 100644 --- a/app/serializers/cluster_entity.rb +++ b/app/serializers/cluster_entity.rb @@ -3,4 +3,5 @@ class ClusterEntity < Grape::Entity expose :status_name, as: :status expose :status_reason + expose :applications, using: ClusterApplicationEntity end diff --git a/app/serializers/cluster_serializer.rb b/app/serializers/cluster_serializer.rb index 2c87202a105..2e13c1501e7 100644 --- a/app/serializers/cluster_serializer.rb +++ b/app/serializers/cluster_serializer.rb @@ -2,6 +2,6 @@ class ClusterSerializer < BaseSerializer entity ClusterEntity def represent_status(resource) - represent(resource, { only: [:status, :status_reason] }) + represent(resource, { only: [:status, :status_reason, :applications] }) end end diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb new file mode 100644 index 00000000000..9a4ce31cb39 --- /dev/null +++ b/app/services/clusters/applications/base_helm_service.rb @@ -0,0 +1,29 @@ +module Clusters + module Applications + class BaseHelmService + attr_accessor :app + + def initialize(app) + @app = app + end + + protected + + def cluster + app.cluster + end + + def kubeclient + cluster.kubeclient + end + + def helm_api + @helm_api ||= Gitlab::Kubernetes::Helm.new(kubeclient) + end + + def install_command + @install_command ||= app.install_command + end + end + end +end diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb new file mode 100644 index 00000000000..bde090eaeec --- /dev/null +++ b/app/services/clusters/applications/check_installation_progress_service.rb @@ -0,0 +1,65 @@ +module Clusters + module Applications + class CheckInstallationProgressService < BaseHelmService + def execute + return unless app.installing? + + case installation_phase + when Gitlab::Kubernetes::Pod::SUCCEEDED + on_success + when Gitlab::Kubernetes::Pod::FAILED + on_failed + else + check_timeout + end + rescue KubeException => ke + app.make_errored!("Kubernetes error: #{ke.message}") unless app.errored? + end + + private + + def on_success + app.make_installed! + ensure + remove_installation_pod + end + + def on_failed + app.make_errored!(installation_errors || 'Installation silently failed') + ensure + remove_installation_pod + end + + def check_timeout + if timeouted? + begin + app.make_errored!('Installation timeouted') + ensure + remove_installation_pod + end + else + ClusterWaitForAppInstallationWorker.perform_in( + ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) + end + end + + def timeouted? + Time.now.utc - app.updated_at.to_time.utc > ClusterWaitForAppInstallationWorker::TIMEOUT + end + + def remove_installation_pod + helm_api.delete_installation_pod!(install_command.pod_name) + rescue + # no-op + end + + def installation_phase + helm_api.installation_status(install_command.pod_name) + end + + def installation_errors + helm_api.installation_log(install_command.pod_name) + end + end + end +end diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb new file mode 100644 index 00000000000..8ceeec687cd --- /dev/null +++ b/app/services/clusters/applications/install_service.rb @@ -0,0 +1,21 @@ +module Clusters + module Applications + class InstallService < BaseHelmService + def execute + return unless app.scheduled? + + begin + app.make_installing! + helm_api.install(install_command) + + ClusterWaitForAppInstallationWorker.perform_in( + ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) + rescue KubeException => ke + app.make_errored!("Kubernetes error: #{ke.message}") + rescue StandardError + app.make_errored!("Can't start installation process") + end + end + end + end +end diff --git a/app/services/clusters/applications/schedule_installation_service.rb b/app/services/clusters/applications/schedule_installation_service.rb new file mode 100644 index 00000000000..eb8caa68ef7 --- /dev/null +++ b/app/services/clusters/applications/schedule_installation_service.rb @@ -0,0 +1,22 @@ +module Clusters + module Applications + class ScheduleInstallationService < ::BaseService + def execute + application_class.find_or_create_by!(cluster: cluster).try do |application| + application.make_scheduled! + ClusterInstallAppWorker.perform_async(application.name, application.id) + end + end + + private + + def application_class + params[:application_class] + end + + def cluster + params[:cluster] + end + end + end +end diff --git a/app/services/delete_merged_branches_service.rb b/app/services/delete_merged_branches_service.rb index 077268b2388..cb235a85daf 100644 --- a/app/services/delete_merged_branches_service.rb +++ b/app/services/delete_merged_branches_service.rb @@ -13,7 +13,7 @@ class DeleteMergedBranchesService < BaseService # Prevent deletion of branches relevant to open merge requests branches -= merge_request_branch_names # Prevent deletion of protected branches - branches = branches.reject { |branch| project.protected_for?(branch) } + branches = branches.reject { |branch| ProtectedBranch.protected?(project, branch) } branches.each do |branch| DeleteBranchService.new(project, current_user).execute(branch) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index 455b302d819..c950da44aba 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -4,6 +4,18 @@ module Projects Error = Class.new(StandardError) + # Returns true if this importer is supposed to perform its work in the + # background. + # + # This method will only return `true` if async importing is explicitly + # supported by an importer class (`Gitlab::GithubImport::ParallelImporter` + # for example). + def async? + return false unless has_importer? + + !!importer_class.try(:async?) + end + def execute add_repository_to_project unless project.gitlab_project_import? @@ -75,12 +87,16 @@ module Projects end end + def importer_class + Gitlab::ImportSources.importer(project.import_type) + end + def has_importer? Gitlab::ImportSources.importer_names.include?(project.import_type) end def importer - Gitlab::ImportSources.importer(project.import_type).new(project) + importer_class.new(project) end def unknown_url? diff --git a/app/views/layouts/header/_default.html.haml b/app/views/layouts/header/_default.html.haml index 5ff6ac5fc00..1eca412aff9 100644 --- a/app/views/layouts/header/_default.html.haml +++ b/app/views/layouts/header/_default.html.haml @@ -61,7 +61,7 @@ = link_to "Help", help_path %li.divider %li - = link_to "Sign out", destroy_user_session_path, method: :delete, class: "sign-out-link" + = link_to "Sign out", destroy_user_session_path, class: "sign-out-link" - if session[:impersonator_id] %li.impersonation = link_to admin_impersonation_path, class: 'impersonation-btn', method: :delete, title: "Stop impersonation", aria: { label: 'Stop impersonation' }, data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } do diff --git a/app/views/projects/clusters/show.html.haml b/app/views/projects/clusters/show.html.haml index ebb9383ca12..b7671f5e3c4 100644 --- a/app/views/projects/clusters/show.html.haml +++ b/app/views/projects/clusters/show.html.haml @@ -4,11 +4,18 @@ - expanded = Rails.env.test? -- status_path = status_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster.id, format: :json) if can?(current_user, :admin_cluster, @cluster) && @cluster.on_creation? +- status_path = status_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster.id, format: :json) if can?(current_user, :admin_cluster, @cluster) .edit-cluster-form.js-edit-cluster-form{ data: { status_path: status_path, + install_helm_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :helm), + install_ingress_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :ingress), toggle_status: @cluster.enabled? ? 'true': 'false', cluster_status: @cluster.status_name, - cluster_status_reason: @cluster.status_reason } } + cluster_status_reason: @cluster.status_reason, + help_path: help_page_path('user/project/clusters/index.md', anchor: 'installing-applications') } } + + + .js-cluster-application-notice + .flash-container %section.settings.no-animate.expanded %h4= s_('ClusterIntegration|Enable cluster integration') @@ -49,7 +56,9 @@ .form-group = field.submit _('Save'), class: 'btn btn-success' - %section.settings.no-animate#js-cluster-details{ class: ('expanded' if expanded) } + .cluster-applications-table#js-cluster-applications + + %section.settings#js-cluster-details .settings-header %h4= s_('ClusterIntegration|Cluster details') %button.btn.js-settings-toggle @@ -59,7 +68,7 @@ .settings-content .form_group.append-bottom-20 - %label.append-bottom-10{ for: 'cluter-name' } + %label.append-bottom-10{ for: 'cluster-name' } = s_('ClusterIntegration|Cluster name') .input-group %input.form-control.cluster-name{ value: @cluster.name, disabled: true } diff --git a/app/views/projects/commit/_commit_box.html.haml b/app/views/projects/commit/_commit_box.html.haml index ff17372fdd9..8b9c1bbb602 100644 --- a/app/views/projects/commit/_commit_box.html.haml +++ b/app/views/projects/commit/_commit_box.html.haml @@ -61,13 +61,13 @@ %span.cgray= n_('parent', 'parents', @commit.parents.count) - @commit.parents.each do |parent| = link_to parent.short_id, project_commit_path(@project, parent), class: "commit-sha" - %span.commit-info.branches + .commit-info.branches %i.fa.fa-spinner.fa-spin - if @commit.last_pipeline - last_pipeline = @commit.last_pipeline .well-segment.pipeline-info - .status-icon-container{ class: "ci-status-icon-#{@commit.status}" } + .status-icon-container{ class: "ci-status-icon-#{last_pipeline.status}" } = link_to project_pipeline_path(@project, last_pipeline.id) do = ci_icon_for_status(last_pipeline.status) #{ _('Pipeline') } diff --git a/app/views/projects/commit/_limit_exceeded_message.html.haml b/app/views/projects/commit/_limit_exceeded_message.html.haml new file mode 100644 index 00000000000..84a52d49487 --- /dev/null +++ b/app/views/projects/commit/_limit_exceeded_message.html.haml @@ -0,0 +1,8 @@ +.has-tooltip{ class: "limit-box limit-box-#{objects} prepend-left-5", data: { title: "Project has too many #{label_for_message} to search"} } + .limit-icon + - if objects == :branch + = icon('code-fork') + - else + = icon('tag') + .limit-message + %span #{label_for_message.capitalize} unavailable diff --git a/app/views/projects/commit/branches.html.haml b/app/views/projects/commit/branches.html.haml index 911c9ddce06..8611129b356 100644 --- a/app/views/projects/commit/branches.html.haml +++ b/app/views/projects/commit/branches.html.haml @@ -1,15 +1,15 @@ -- if @branches.any? || @tags.any? +- if @branches_limit_exceeded + = render 'limit_exceeded_message', objects: :branch, label_for_message: "branches" +- elsif @branches.any? - branch = commit_default_branch(@project, @branches) - = link_to(project_ref_path(@project, branch), class: "label label-gray ref-name") do - = icon('code-fork') - = branch + = commit_branch_link(project_ref_path(@project, branch), branch) - -# `commit_default_branch` deletes the default branch from `@branches`, - -# so only render this if we have more branches left - - if @branches.any? || @tags.any? - %span - = link_to "…", "#", class: "js-details-expand label label-gray" - - %span.js-details-content.hide - = commit_branches_links(@project, @branches) if @branches.any? - = commit_tags_links(@project, @tags) if @tags.any? +- if @branches.any? || @tags.any? || @tags_limit_exceeded + %span + = link_to "…", "#", class: "js-details-expand label label-gray" + %span.js-details-content.hide + = commit_branches_links(@project, @branches) + - if @tags_limit_exceeded + = render 'limit_exceeded_message', objects: :tag, label_for_message: "tags" + - else + = commit_tags_links(@project, @tags) diff --git a/app/views/projects/issues/_nav_btns.html.haml b/app/views/projects/issues/_nav_btns.html.haml index 13809da6523..0d39edb7bfd 100644 --- a/app/views/projects/issues/_nav_btns.html.haml +++ b/app/views/projects/issues/_nav_btns.html.haml @@ -3,8 +3,8 @@ - if @can_bulk_update = button_tag "Edit issues", class: "btn btn-default append-right-10 js-bulk-update-toggle" = link_to "New issue", new_project_issue_path(@project, - issue: { assignee_id: issues_finder.assignee.try(:id), - milestone_id: issues_finder.milestones.first.try(:id) }), + issue: { assignee_id: finder.assignee.try(:id), + milestone_id: finder.milestones.first.try(:id) }), class: "btn btn-new", title: "New issue", id: "new_issue_link" diff --git a/app/views/projects/tags/new.html.haml b/app/views/projects/tags/new.html.haml index 3e99e0e8234..031efa903c5 100644 --- a/app/views/projects/tags/new.html.haml +++ b/app/views/projects/tags/new.html.haml @@ -33,7 +33,7 @@ = s_('TagsPage|Optionally, add a message to the tag.') %hr .form-group - = label_tag :release_description, 'Release notes', class: 'control-label' + = label_tag :release_description, s_('TagsPage|Release notes'), class: 'control-label' .col-sm-10 = render layout: 'projects/md_preview', locals: { url: preview_markdown_path(@project), referenced_users: true } do = render 'projects/zen', attr: :release_description, classes: 'note-textarea', placeholder: s_('TagsPage|Write your release notes or drag files here...'), current_text: @release_description @@ -41,6 +41,6 @@ .help-block = s_('TagsPage|Optionally, add release notes to the tag. They will be stored in the GitLab database and displayed on the tags page.') .form-actions - = button_tag 'Create tag', class: 'btn btn-create', tabindex: 3 - = link_to 'Cancel', project_tags_path(@project), class: 'btn btn-cancel' + = button_tag s_('TagsPage|Create tag'), class: 'btn btn-create', tabindex: 3 + = link_to s_('TagsPage|Cancel'), project_tags_path(@project), class: 'btn btn-cancel' %script#availableRefs{ type: "application/json" }= @project.repository.ref_names.to_json.html_safe diff --git a/app/views/shared/icons/_icon_autodevops.svg b/app/views/shared/icons/_icon_autodevops.svg index 423ca6d760d..7e47c084bde 100644 --- a/app/views/shared/icons/_icon_autodevops.svg +++ b/app/views/shared/icons/_icon_autodevops.svg @@ -29,7 +29,7 @@ </g> <g fill-rule="nonzero" transform="rotate(15 -315.035 277.714)"> <path fill="#FFFFFF" d="M12.275,10.57 C13.986216,9.15630755 15.921048,8.03765363 18,7.26 L18,5.5 C18,2.463 20.47,0 23.493,0 L26.507,0 C27.9648848,0.000530018716 29.3628038,0.580386367 30.3930274,1.61192286 C31.4232511,2.64345935 32.0013267,4.04211574 32,5.5 L32,7.26 C34.098,8.043 36.03,9.17 37.725,10.57 L39.253,9.688 C41.8816141,8.17268496 45.2407537,9.07039379 46.763,11.695 L48.27,14.305 C48.9984289,15.5678669 49.1951495,17.0684426 48.8168566,18.4763972 C48.4385638,19.8843518 47.5162683,21.0842673 46.253,21.812 L44.728,22.693 C44.907,23.769 45,24.873 45,26 C45,27.127 44.907,28.231 44.728,29.307 L46.253,30.187 C48.8800379,31.705769 49.7822744,35.0642181 48.27,37.695 L46.763,40.305 C46.0335844,41.5673849 44.8323832,42.4881439 43.4238487,42.8645658 C42.0153143,43.2409877 40.5149245,43.0422119 39.253,42.312 L37.725,41.43 C36.013784,42.8436924 34.078952,43.9623464 32,44.74 L32,46.5 C32,49.537 29.53,52 26.507,52 L23.493,52 C22.0351152,51.99947 20.6371962,51.4196136 19.6069726,50.3880771 C18.5767489,49.3565406 17.9986733,47.9578843 18,46.5 L18,44.74 C15.921048,43.9623464 13.986216,42.8436924 12.275,41.43 L10.747,42.312 C8.11838594,43.827315 4.75924629,42.9296062 3.237,40.305 L1.73,37.695 C1.00157113,36.4321331 0.804850523,34.9315574 1.18314337,33.5236028 C1.56143621,32.1156482 2.48373172,30.9157327 3.747,30.188 L5.272,29.307 C5.09051204,28.2140265 4.9995366,27.107939 5,26 C5,24.873 5.093,23.769 5.272,22.693 L3.747,21.813 C1.11996213,20.294231 0.217725591,16.9357819 1.73,14.305 L3.237,11.695 C3.96641559,10.4326151 5.16761682,9.51185609 6.57615125,9.13543417 C7.98468568,8.75901226 9.48507553,8.95778814 10.747,9.688 L12.275,10.57 Z"/> - <path class="animated spin-cw infinite" fill="#E1DBF1" d="M17.9996486,7.25963195 L18.0000013,5.49772675 C18.0034459,2.46713881 20.4561478,0.00952173148 23.493,0 L26.507,0 C29.542757,0 32,2.46161709 32,5.5 L32,7.25850184 C34.0799663,8.03664754 36.0149544,9.15559094 37.7260175,10.5694605 L39.2547869,9.68691874 C41.8812087,8.17416302 45.2363972,9.06948854 46.7630175,11.6949424 L48.270687,14.3061027 C48.9989901,15.569417 49.1952874,17.0704122 48.816349,18.4785295 C48.4374106,19.8866468 47.5143145,21.0864021 46.2530682,21.8120114 L44.7278655,22.6926677 C44.9091017,23.7802451 45,24.8850821 45,26 C45,27.1144218 44.9091826,28.218078 44.7278653,29.3073326 L46.2547984,30.1889888 C48.8778516,31.7070439 49.7801588,35.0599752 48.2700175,37.6950576 L46.7625317,40.3058986 C46.0327098,41.5684739 44.8309328,42.4891542 43.4219037,42.8651509 C42.0128746,43.2411475 40.512172,43.0416186 39.2533538,42.312255 L37.7244858,41.4299789 C36.013753,42.8435912 34.0794396,43.9622923 32.0003514,44.7403681 L31.9999987,46.5022733 C31.9965541,49.5328612 29.5438522,51.9904783 26.507,52 L23.493,52 C20.457243,52 18,49.5383829 18,46.5 L18,44.7414988 C15.9200337,43.9633525 13.9850456,42.8444091 12.2739825,41.4305395 L10.7452131,42.3130813 C8.11879127,43.825837 4.76360277,42.9305115 3.23698247,40.3050576 L1.72931303,37.6938973 C1.0010099,36.430583 0.804712603,34.9295878 1.18365098,33.5214705 C1.56258936,32.1133532 2.48568546,30.9135979 3.74693178,30.1879886 L5.27213454,29.3073323 C5.09089825,28.2197549 5,27.114918 5.00000019,26.0008761 C4.99951488,24.8930059 5.0904571,23.7869854 5.27213502,22.6926675 L3.74520157,21.8110112 C1.12214836,20.2929561 0.219841192,16.9400248 1.72998247,14.3049424 L3.23746831,11.6941014 C3.96729024,10.4315261 5.16906725,9.51084579 6.5780963,9.13484913 C7.98712536,8.75885247 9.48782803,8.95838137 10.7466462,9.687745 L12.2748018,10.56961 C14.0209791,9.13635584 15.9392199,8.03072455 17.9996486,7.25963195 Z M13.7518374,14.537862 C13.108069,15.069723 12.2016163,15.1456339 11.4783538,14.728255 L8.74433999,13.1505123 C8.40103903,12.9516035 7.99274958,12.8973186 7.60940137,12.9996143 C7.22605315,13.10191 6.89909107,13.3523954 6.70101753,13.6950576 L5.19724591,16.2994454 C4.78547321,17.0179634 5.03203388,17.9341714 5.74706822,18.3479886 L8.47306822,19.9219886 C9.19530115,20.3390079 9.58295216,21.1604138 9.44574883,21.983032 L9.21798321,23.3486236 C9.07251948,24.2246212 8.99961081,25.111131 9,26 C9,26.8953847 9.0728258,27.7804297 9.21774883,28.649968 L9.44574883,30.016968 C9.58295216,30.8395862 9.19530115,31.6609921 8.47306822,32.0780114 L5.74435077,33.6535776 C5.40046982,33.851417 5.14932721,34.1778291 5.04623114,34.5609292 C4.94313508,34.9440294 4.9965408,35.3523984 5.19401753,35.6949424 L6.69795587,38.2996585 C7.11427713,39.0156351 8.03110189,39.260288 8.7470791,38.8479035 L11.4770791,37.2719035 C12.200376,36.8543519 13.1069795,36.9302031 13.7508374,37.462138 L14.8210499,38.3463136 C16.1898549,39.4774943 17.737648,40.3725891 19.3990866,40.9941596 L20.6990866,41.4791596 C21.4813437,41.7710017 22,42.5180761 22,43.353 L22,46.5 C22,47.3308348 22.6679761,48 23.493,48 L26.5007228,48.0000099 C27.328845,47.9974107 27.99906,47.3258525 28,46.5 L28,43.353 C28,42.5185702 28.5180515,41.771829 29.2996486,41.4796319 L30.599003,40.9938734 C32.261836,40.3715765 33.8093225,39.4764853 35.1790197,38.3444304 L36.2490197,37.4614304 C36.8927697,36.9301861 37.798736,36.8545694 38.5216462,37.271745 L41.25566,38.8494877 C41.598961,39.0483965 42.0072504,39.1026814 42.3905986,39.0003857 C42.7739468,38.89809 43.1009089,38.6476046 43.2989825,38.3049424 L44.8027541,35.7005546 C45.2145268,34.9820366 44.9679661,34.0658286 44.2529318,33.6520114 L41.5269318,32.0780114 C40.8046988,31.6609921 40.4170478,30.8395862 40.5542512,30.016968 L40.7821577,28.6505288 C40.9272286,27.7792134 41,26.8950523 41,26 C41,25.1046153 40.9271742,24.2195703 40.7822512,23.350032 L40.5542512,21.983032 C40.4170478,21.1604138 40.8046988,20.3390079 41.5269318,19.9219886 L44.2556492,18.3464224 C44.5995302,18.148583 44.8506728,17.8221709 44.9537689,17.4390708 C45.0568649,17.0559706 45.0034592,16.6476016 44.8059825,16.3050576 L43.3020441,13.7003415 C42.8857229,12.9843649 41.9688981,12.739712 41.2529209,13.1520965 L38.5229209,14.7280965 C37.799624,15.1456481 36.8930205,15.0697969 36.2491626,14.537862 L35.1789501,13.6536864 C33.8101451,12.5225057 32.262352,11.6274109 30.6021792,11.0063122 L29.3021792,10.5223122 C28.5192618,10.230826 28,9.48341836 28,8.648 L28,5.5 C28,4.66916515 27.3320239,4 26.507,4 L23.4992772,3.99999015 C22.671155,4.00258933 22.00094,4.67414748 22,5.5 L22,8.647 C22,9.48142977 21.4819485,10.228171 20.7003514,10.5203681 L19.400997,11.0061266 C17.738164,11.6284235 16.1906775,12.5235147 14.822142,13.6546103 C14.8121128,13.6628994 14.4553446,13.9573166 13.7518374,14.537862 Z"/> + <path fill="#E1DBF1" d="M17.9996486,7.25963195 L18.0000013,5.49772675 C18.0034459,2.46713881 20.4561478,0.00952173148 23.493,0 L26.507,0 C29.542757,0 32,2.46161709 32,5.5 L32,7.25850184 C34.0799663,8.03664754 36.0149544,9.15559094 37.7260175,10.5694605 L39.2547869,9.68691874 C41.8812087,8.17416302 45.2363972,9.06948854 46.7630175,11.6949424 L48.270687,14.3061027 C48.9989901,15.569417 49.1952874,17.0704122 48.816349,18.4785295 C48.4374106,19.8866468 47.5143145,21.0864021 46.2530682,21.8120114 L44.7278655,22.6926677 C44.9091017,23.7802451 45,24.8850821 45,26 C45,27.1144218 44.9091826,28.218078 44.7278653,29.3073326 L46.2547984,30.1889888 C48.8778516,31.7070439 49.7801588,35.0599752 48.2700175,37.6950576 L46.7625317,40.3058986 C46.0327098,41.5684739 44.8309328,42.4891542 43.4219037,42.8651509 C42.0128746,43.2411475 40.512172,43.0416186 39.2533538,42.312255 L37.7244858,41.4299789 C36.013753,42.8435912 34.0794396,43.9622923 32.0003514,44.7403681 L31.9999987,46.5022733 C31.9965541,49.5328612 29.5438522,51.9904783 26.507,52 L23.493,52 C20.457243,52 18,49.5383829 18,46.5 L18,44.7414988 C15.9200337,43.9633525 13.9850456,42.8444091 12.2739825,41.4305395 L10.7452131,42.3130813 C8.11879127,43.825837 4.76360277,42.9305115 3.23698247,40.3050576 L1.72931303,37.6938973 C1.0010099,36.430583 0.804712603,34.9295878 1.18365098,33.5214705 C1.56258936,32.1133532 2.48568546,30.9135979 3.74693178,30.1879886 L5.27213454,29.3073323 C5.09089825,28.2197549 5,27.114918 5.00000019,26.0008761 C4.99951488,24.8930059 5.0904571,23.7869854 5.27213502,22.6926675 L3.74520157,21.8110112 C1.12214836,20.2929561 0.219841192,16.9400248 1.72998247,14.3049424 L3.23746831,11.6941014 C3.96729024,10.4315261 5.16906725,9.51084579 6.5780963,9.13484913 C7.98712536,8.75885247 9.48782803,8.95838137 10.7466462,9.687745 L12.2748018,10.56961 C14.0209791,9.13635584 15.9392199,8.03072455 17.9996486,7.25963195 Z M13.7518374,14.537862 C13.108069,15.069723 12.2016163,15.1456339 11.4783538,14.728255 L8.74433999,13.1505123 C8.40103903,12.9516035 7.99274958,12.8973186 7.60940137,12.9996143 C7.22605315,13.10191 6.89909107,13.3523954 6.70101753,13.6950576 L5.19724591,16.2994454 C4.78547321,17.0179634 5.03203388,17.9341714 5.74706822,18.3479886 L8.47306822,19.9219886 C9.19530115,20.3390079 9.58295216,21.1604138 9.44574883,21.983032 L9.21798321,23.3486236 C9.07251948,24.2246212 8.99961081,25.111131 9,26 C9,26.8953847 9.0728258,27.7804297 9.21774883,28.649968 L9.44574883,30.016968 C9.58295216,30.8395862 9.19530115,31.6609921 8.47306822,32.0780114 L5.74435077,33.6535776 C5.40046982,33.851417 5.14932721,34.1778291 5.04623114,34.5609292 C4.94313508,34.9440294 4.9965408,35.3523984 5.19401753,35.6949424 L6.69795587,38.2996585 C7.11427713,39.0156351 8.03110189,39.260288 8.7470791,38.8479035 L11.4770791,37.2719035 C12.200376,36.8543519 13.1069795,36.9302031 13.7508374,37.462138 L14.8210499,38.3463136 C16.1898549,39.4774943 17.737648,40.3725891 19.3990866,40.9941596 L20.6990866,41.4791596 C21.4813437,41.7710017 22,42.5180761 22,43.353 L22,46.5 C22,47.3308348 22.6679761,48 23.493,48 L26.5007228,48.0000099 C27.328845,47.9974107 27.99906,47.3258525 28,46.5 L28,43.353 C28,42.5185702 28.5180515,41.771829 29.2996486,41.4796319 L30.599003,40.9938734 C32.261836,40.3715765 33.8093225,39.4764853 35.1790197,38.3444304 L36.2490197,37.4614304 C36.8927697,36.9301861 37.798736,36.8545694 38.5216462,37.271745 L41.25566,38.8494877 C41.598961,39.0483965 42.0072504,39.1026814 42.3905986,39.0003857 C42.7739468,38.89809 43.1009089,38.6476046 43.2989825,38.3049424 L44.8027541,35.7005546 C45.2145268,34.9820366 44.9679661,34.0658286 44.2529318,33.6520114 L41.5269318,32.0780114 C40.8046988,31.6609921 40.4170478,30.8395862 40.5542512,30.016968 L40.7821577,28.6505288 C40.9272286,27.7792134 41,26.8950523 41,26 C41,25.1046153 40.9271742,24.2195703 40.7822512,23.350032 L40.5542512,21.983032 C40.4170478,21.1604138 40.8046988,20.3390079 41.5269318,19.9219886 L44.2556492,18.3464224 C44.5995302,18.148583 44.8506728,17.8221709 44.9537689,17.4390708 C45.0568649,17.0559706 45.0034592,16.6476016 44.8059825,16.3050576 L43.3020441,13.7003415 C42.8857229,12.9843649 41.9688981,12.739712 41.2529209,13.1520965 L38.5229209,14.7280965 C37.799624,15.1456481 36.8930205,15.0697969 36.2491626,14.537862 L35.1789501,13.6536864 C33.8101451,12.5225057 32.262352,11.6274109 30.6021792,11.0063122 L29.3021792,10.5223122 C28.5192618,10.230826 28,9.48341836 28,8.648 L28,5.5 C28,4.66916515 27.3320239,4 26.507,4 L23.4992772,3.99999015 C22.671155,4.00258933 22.00094,4.67414748 22,5.5 L22,8.647 C22,9.48142977 21.4819485,10.228171 20.7003514,10.5203681 L19.400997,11.0061266 C17.738164,11.6284235 16.1906775,12.5235147 14.822142,13.6546103 C14.8121128,13.6628994 14.4553446,13.9573166 13.7518374,14.537862 Z"/> <g transform="rotate(15 -59.137 82.348)"> <circle cx="8" cy="8" r="8" fill="#FFFFFF" transform="translate(.035 6.008)"/> <path fill="#6B4FBB" d="M7.40192379,14.7679492 C2.98364579,14.7679492 -0.598076211,11.1862272 -0.598076211,6.76794919 C-0.598076211,2.34967119 2.98364579,-1.23205081 7.40192379,-1.23205081 C11.8202018,-1.23205081 15.4019238,2.34967119 15.4019238,6.76794919 C15.4019238,11.1862272 11.8202018,14.7679492 7.40192379,14.7679492 Z M7.40192379,10.7679492 C9.61106279,10.7679492 11.4019238,8.97708819 11.4019238,6.76794919 C11.4019238,4.55881019 9.61106279,2.76794919 7.40192379,2.76794919 C5.19278479,2.76794919 3.40192379,4.55881019 3.40192379,6.76794919 C3.40192379,8.97708819 5.19278479,10.7679492 7.40192379,10.7679492 Z"/> @@ -37,7 +37,7 @@ </g> <g fill-rule="nonzero" transform="rotate(15 -402.968 460.884)"> <path fill="#FFFFFF" d="M9.82,8.53730769 C11.1889728,7.39547918 12.7368384,6.49195101 14.4,5.86384615 L14.4,4.44230769 C14.4,1.98934615 16.376,0 18.7944,0 L21.2056,0 C22.3719078,0.00042809204 23.4902431,0.468773604 24.314422,1.30193769 C25.1386009,2.13510179 25.6010613,3.26478579 25.6,4.44230769 L25.6,5.86384615 C27.2784,6.49626923 28.824,7.40653846 30.18,8.53730769 L31.4024,7.82492308 C33.5052912,6.60101478 36.192603,7.32608729 37.4104,9.44596154 L38.616,11.5540385 C39.1987431,12.5740464 39.3561196,13.7860498 39.0534853,14.9232439 C38.750851,16.060438 38.0130146,17.0296006 37.0024,17.6173846 L35.7824,18.3289615 C35.9256,19.1980385 36,20.0897308 36,21 C36,21.9102692 35.9256,22.8019615 35.7824,23.6710385 L37.0024,24.3818077 C39.1040303,25.6085057 39.8258195,28.3210992 38.616,30.4459615 L37.4104,32.5540385 C36.8268675,33.573657 35.8659065,34.317347 34.739079,34.6213801 C33.6122515,34.9254132 32.4119396,34.7648634 31.4024,34.1750769 L30.18,33.4626923 C28.8110272,34.6045208 27.2631616,35.508049 25.6,36.1361538 L25.6,37.5576923 C25.6,40.0106538 23.624,42 21.2056,42 L18.7944,42 C17.6280922,41.9995719 16.5097569,41.5312264 15.685578,40.6980623 C14.8613991,39.8648982 14.3989387,38.7352142 14.4,37.5576923 L14.4,36.1361538 C12.7368384,35.508049 11.1889728,34.6045208 9.82,33.4626923 L8.5976,34.1750769 C6.49470875,35.3989852 3.80739703,34.6739127 2.5896,32.5540385 L1.384,30.4459615 C0.8012569,29.4259536 0.643880418,28.2139502 0.946514692,27.0767561 C1.24914897,25.939562 1.98698538,24.9703994 2.9976,24.3826154 L4.2176,23.6710385 C4.07240963,22.7882521 3.99962928,21.8948738 4,21 C4,20.0897308 4.0744,19.1980385 4.2176,18.3289615 L2.9976,17.6181923 C0.895969702,16.3914943 0.174180473,13.6789008 1.384,11.5540385 L2.5896,9.44596154 C3.17313247,8.42634297 4.13409345,7.682653 5.260921,7.37861991 C6.38774855,7.07458682 7.58806043,7.23513658 8.5976,7.82492308 L9.82,8.53730769 Z"/> - <path class="animated spin-ccw infinite" fill="#FEE1D3" d="M14.0000007,5.6038043 L14.0000013,4.44005609 C14.0029906,1.78475013 16.1390906,-0.376211234 18.7944,-0.384615385 L21.2056,-0.384615385 C23.8595941,-0.384615385 26,1.78021801 26,4.44230769 L26,5.60295806 C27.5208716,6.20655954 28.9434678,7.03621848 30.2204219,8.06411282 L31.1970056,7.49492104 C33.4941909,6.15907529 36.4301298,6.95005805 37.7609369,9.26076474 L38.9671983,11.3699991 C39.5988409,12.4761812 39.768854,13.7886936 39.4405746,15.0202941 C39.1116282,16.2543969 38.308799,17.3078735 37.2096539,17.946304 L36.2175721,18.5246428 C36.3390841,19.3401617 36.4,20.1667594 36.4,21 C36.4,21.8329668 36.339124,22.6588262 36.2175401,23.4753391 L37.2113882,24.0547082 C39.4944154,25.3886826 40.276605,28.3232105 38.9665369,30.6311583 L37.7604568,32.7400742 C37.1252608,33.8495148 36.0768547,34.6604208 34.8452776,34.9922248 C33.6111681,35.324711 32.2964469,35.1482289 31.195569,34.5042428 L30.2192355,33.9354047 C28.9426535,34.9630196 27.5206806,35.7924453 25.9999993,36.3961957 L25.9999987,37.5599439 C25.9970094,40.2152499 23.8609094,42.3762112 21.2056,42.3846154 L18.7944,42.3846154 C16.1404059,42.3846154 14,40.219782 14,37.5576923 L14,36.3970419 C12.4791284,35.7934405 11.0565322,34.9637815 9.77957815,33.9358872 L8.80299442,34.505079 C6.50580915,35.8409247 3.56987021,35.049942 2.23906313,32.7392353 L1.03280169,30.6300009 C0.401159146,29.5238188 0.231145999,28.2113064 0.559425405,26.9797059 C0.888371786,25.7456031 1.69120101,24.6921265 2.79034606,24.053696 L3.78242779,23.4753573 C3.66091587,22.6598457 3.60000002,21.8333228 3.60000019,21.0008678 C3.59964068,20.1722851 3.66061719,19.3449468 3.78254167,18.5247085 L2.78861183,17.9452918 C0.505584602,16.6113174 -0.276605002,13.6767895 1.03346313,11.3688417 L2.23954317,9.25992583 C2.87473915,8.15048519 3.92314533,7.33957919 5.15472238,7.00777521 C6.38883187,6.67528896 7.70355311,6.85177112 8.80443097,7.49575721 L9.78076186,8.06459377 C11.0573465,7.03698045 12.4793194,6.20755475 14.0000007,5.6038043 Z M11.2634746,12.0326234 C10.617233,12.5716613 9.7026973,12.6485026 8.97556903,12.2248582 L6.78774825,10.9501716 C6.60754053,10.8447551 6.39506809,10.8162338 6.19527576,10.8700606 C5.99295099,10.9245697 5.8183659,11.0596053 5.71133687,11.246543 L4.50892658,13.3490215 C4.28085652,13.7508163 4.41776119,14.2644394 4.80485394,14.4906191 L6.98565394,15.7619268 C7.70254629,16.1798426 8.08690703,16.9970357 7.95165511,17.8157512 L7.76948523,18.9184706 C7.65638664,19.6061109 7.59969735,20.3020342 7.6,21 C7.6,21.7031066 7.65662064,22.3978283 7.76925511,23.0801334 L7.95165511,24.1842488 C8.08690703,25.0029643 7.70254629,25.8201574 6.98565394,26.2380732 L4.80213007,27.5109659 C4.61772321,27.6180778 4.48116147,27.7972748 4.42448029,28.0099246 C4.36713215,28.2250767 4.39688141,28.454743 4.50573687,28.6453801 L5.70831165,30.7481858 C5.93243371,31.1373303 6.41410538,31.2670993 6.79049373,31.0482253 L8.97449373,29.7753023 C9.7016554,29.3514832 10.6163433,29.4282639 11.2626746,29.9673766 L12.1188867,30.6815536 C13.1796505,31.566598 14.3786867,32.2666727 15.6649769,32.7525215 L16.7049769,33.1442523 C17.4841581,33.4377419 18,34.1832625 18,35.0158846 L18,37.5576923 C18,38.02074 18.3597694,38.3846154 18.7944,38.3846154 L21.1992624,38.3846254 C21.6372484,38.3832375 21.9994819,38.0167881 22,37.5576923 L22,35.0158846 C22,34.18376 22.5152346,33.4385758 23.2937506,33.1447321 L24.3331012,32.7524389 C25.620867,32.2658727 26.8196661,31.5658006 27.8813806,30.679856 L28.7373806,29.9666637 C29.3836087,29.4282468 30.2976553,29.3517028 31.024431,29.7751418 L33.2122517,31.0498284 C33.3924595,31.1552449 33.6049319,31.1837662 33.8047242,31.1299394 C34.007049,31.0754303 34.1816341,30.9403947 34.2886631,30.753457 L35.4910734,28.6509785 C35.7191435,28.2491837 35.5822388,27.7355606 35.1951461,27.5093809 L33.0143461,26.2380732 C32.2974537,25.8201574 31.913093,25.0029643 32.0483449,24.1842488 L32.2306531,23.0806893 C32.3434217,22.3968737 32.4,21.7028459 32.4,21 C32.4,20.2968934 32.3433794,19.6021717 32.2307449,18.9198666 L32.0483449,17.8157512 C31.913093,16.9970357 32.2974537,16.1798426 33.0143461,15.7619268 L35.1978699,14.4890341 C35.3822768,14.3819222 35.5188385,14.2027252 35.5755197,13.9900754 C35.6328679,13.7749233 35.6031186,13.545257 35.4942631,13.3546199 L34.2916883,11.2518142 C34.0675663,10.8626697 33.5858946,10.7329007 33.2095063,10.9517747 L31.0255063,12.2246977 C30.2983446,12.6485168 29.3836567,12.5717361 28.7373254,12.0326234 L27.8811133,11.3184464 C26.8203495,10.433402 25.6213133,9.73332732 24.3362966,9.24795765 L23.2962966,8.85703457 C22.5164499,8.56389992 22,7.81804293 22,6.98492308 L22,4.44230769 C22,3.97925995 21.6402306,3.61538462 21.2056,3.61538462 L18.8007376,3.61537457 C18.3627516,3.61676247 18.0005181,3.98321188 18,4.44230769 L18,6.98411538 C18,7.81623999 17.4847654,8.56142419 16.7062494,8.85526793 L15.6668988,9.24756113 C14.379133,9.73412728 13.1803339,10.4341994 12.1197785,11.3191775 C12.1108094,11.3266617 11.8253748,11.564477 11.2634746,12.0326234 Z"/> + <path fill="#FEE1D3" d="M14.0000007,5.6038043 L14.0000013,4.44005609 C14.0029906,1.78475013 16.1390906,-0.376211234 18.7944,-0.384615385 L21.2056,-0.384615385 C23.8595941,-0.384615385 26,1.78021801 26,4.44230769 L26,5.60295806 C27.5208716,6.20655954 28.9434678,7.03621848 30.2204219,8.06411282 L31.1970056,7.49492104 C33.4941909,6.15907529 36.4301298,6.95005805 37.7609369,9.26076474 L38.9671983,11.3699991 C39.5988409,12.4761812 39.768854,13.7886936 39.4405746,15.0202941 C39.1116282,16.2543969 38.308799,17.3078735 37.2096539,17.946304 L36.2175721,18.5246428 C36.3390841,19.3401617 36.4,20.1667594 36.4,21 C36.4,21.8329668 36.339124,22.6588262 36.2175401,23.4753391 L37.2113882,24.0547082 C39.4944154,25.3886826 40.276605,28.3232105 38.9665369,30.6311583 L37.7604568,32.7400742 C37.1252608,33.8495148 36.0768547,34.6604208 34.8452776,34.9922248 C33.6111681,35.324711 32.2964469,35.1482289 31.195569,34.5042428 L30.2192355,33.9354047 C28.9426535,34.9630196 27.5206806,35.7924453 25.9999993,36.3961957 L25.9999987,37.5599439 C25.9970094,40.2152499 23.8609094,42.3762112 21.2056,42.3846154 L18.7944,42.3846154 C16.1404059,42.3846154 14,40.219782 14,37.5576923 L14,36.3970419 C12.4791284,35.7934405 11.0565322,34.9637815 9.77957815,33.9358872 L8.80299442,34.505079 C6.50580915,35.8409247 3.56987021,35.049942 2.23906313,32.7392353 L1.03280169,30.6300009 C0.401159146,29.5238188 0.231145999,28.2113064 0.559425405,26.9797059 C0.888371786,25.7456031 1.69120101,24.6921265 2.79034606,24.053696 L3.78242779,23.4753573 C3.66091587,22.6598457 3.60000002,21.8333228 3.60000019,21.0008678 C3.59964068,20.1722851 3.66061719,19.3449468 3.78254167,18.5247085 L2.78861183,17.9452918 C0.505584602,16.6113174 -0.276605002,13.6767895 1.03346313,11.3688417 L2.23954317,9.25992583 C2.87473915,8.15048519 3.92314533,7.33957919 5.15472238,7.00777521 C6.38883187,6.67528896 7.70355311,6.85177112 8.80443097,7.49575721 L9.78076186,8.06459377 C11.0573465,7.03698045 12.4793194,6.20755475 14.0000007,5.6038043 Z M11.2634746,12.0326234 C10.617233,12.5716613 9.7026973,12.6485026 8.97556903,12.2248582 L6.78774825,10.9501716 C6.60754053,10.8447551 6.39506809,10.8162338 6.19527576,10.8700606 C5.99295099,10.9245697 5.8183659,11.0596053 5.71133687,11.246543 L4.50892658,13.3490215 C4.28085652,13.7508163 4.41776119,14.2644394 4.80485394,14.4906191 L6.98565394,15.7619268 C7.70254629,16.1798426 8.08690703,16.9970357 7.95165511,17.8157512 L7.76948523,18.9184706 C7.65638664,19.6061109 7.59969735,20.3020342 7.6,21 C7.6,21.7031066 7.65662064,22.3978283 7.76925511,23.0801334 L7.95165511,24.1842488 C8.08690703,25.0029643 7.70254629,25.8201574 6.98565394,26.2380732 L4.80213007,27.5109659 C4.61772321,27.6180778 4.48116147,27.7972748 4.42448029,28.0099246 C4.36713215,28.2250767 4.39688141,28.454743 4.50573687,28.6453801 L5.70831165,30.7481858 C5.93243371,31.1373303 6.41410538,31.2670993 6.79049373,31.0482253 L8.97449373,29.7753023 C9.7016554,29.3514832 10.6163433,29.4282639 11.2626746,29.9673766 L12.1188867,30.6815536 C13.1796505,31.566598 14.3786867,32.2666727 15.6649769,32.7525215 L16.7049769,33.1442523 C17.4841581,33.4377419 18,34.1832625 18,35.0158846 L18,37.5576923 C18,38.02074 18.3597694,38.3846154 18.7944,38.3846154 L21.1992624,38.3846254 C21.6372484,38.3832375 21.9994819,38.0167881 22,37.5576923 L22,35.0158846 C22,34.18376 22.5152346,33.4385758 23.2937506,33.1447321 L24.3331012,32.7524389 C25.620867,32.2658727 26.8196661,31.5658006 27.8813806,30.679856 L28.7373806,29.9666637 C29.3836087,29.4282468 30.2976553,29.3517028 31.024431,29.7751418 L33.2122517,31.0498284 C33.3924595,31.1552449 33.6049319,31.1837662 33.8047242,31.1299394 C34.007049,31.0754303 34.1816341,30.9403947 34.2886631,30.753457 L35.4910734,28.6509785 C35.7191435,28.2491837 35.5822388,27.7355606 35.1951461,27.5093809 L33.0143461,26.2380732 C32.2974537,25.8201574 31.913093,25.0029643 32.0483449,24.1842488 L32.2306531,23.0806893 C32.3434217,22.3968737 32.4,21.7028459 32.4,21 C32.4,20.2968934 32.3433794,19.6021717 32.2307449,18.9198666 L32.0483449,17.8157512 C31.913093,16.9970357 32.2974537,16.1798426 33.0143461,15.7619268 L35.1978699,14.4890341 C35.3822768,14.3819222 35.5188385,14.2027252 35.5755197,13.9900754 C35.6328679,13.7749233 35.6031186,13.545257 35.4942631,13.3546199 L34.2916883,11.2518142 C34.0675663,10.8626697 33.5858946,10.7329007 33.2095063,10.9517747 L31.0255063,12.2246977 C30.2983446,12.6485168 29.3836567,12.5717361 28.7373254,12.0326234 L27.8811133,11.3184464 C26.8203495,10.433402 25.6213133,9.73332732 24.3362966,9.24795765 L23.2962966,8.85703457 C22.5164499,8.56389992 22,7.81804293 22,6.98492308 L22,4.44230769 C22,3.97925995 21.6402306,3.61538462 21.2056,3.61538462 L18.8007376,3.61537457 C18.3627516,3.61676247 18.0005181,3.98321188 18,4.44230769 L18,6.98411538 C18,7.81623999 17.4847654,8.56142419 16.7062494,8.85526793 L15.6668988,9.24756113 C14.379133,9.73412728 13.1803339,10.4341994 12.1197785,11.3191775 C12.1108094,11.3266617 11.8253748,11.564477 11.2634746,12.0326234 Z"/> <g transform="rotate(15 -47.892 66.043)"> <ellipse cx="6.4" cy="6.462" fill="#FFFFFF" rx="6.4" ry="6.462" transform="translate(.028 4.853)"/> <path fill="#FC6D26" d="M5.92153903,11.9125743 C2.3834711,11.9125743 -0.478460969,9.0231237 -0.478460969,5.4664205 C-0.478460969,1.9097173 2.3834711,-0.979733345 5.92153903,-0.979733345 C9.45960696,-0.979733345 12.321539,1.9097173 12.321539,5.4664205 C12.321539,9.0231237 9.45960696,11.9125743 5.92153903,11.9125743 Z M5.92153903,8.71257435 C7.6854047,8.71257435 9.12153903,7.26263103 9.12153903,5.4664205 C9.12153903,3.67020997 7.6854047,2.22026666 5.92153903,2.22026666 C4.15767337,2.22026666 2.72153903,3.67020997 2.72153903,5.4664205 C2.72153903,7.26263103 4.15767337,8.71257435 5.92153903,8.71257435 Z"/> diff --git a/app/views/shared/issuable/_filter.html.haml b/app/views/shared/issuable/_filter.html.haml index d3f0aa2d339..8442d7ff4a2 100644 --- a/app/views/shared/issuable/_filter.html.haml +++ b/app/views/shared/issuable/_filter.html.haml @@ -1,4 +1,3 @@ -- finder = controller.controller_name == 'issues' ? issues_finder : merge_requests_finder - boards_page = controller.controller_name == 'boards' .issues-filters diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb new file mode 100644 index 00000000000..899aed904e4 --- /dev/null +++ b/app/workers/cluster_install_app_worker.rb @@ -0,0 +1,11 @@ +class ClusterInstallAppWorker + include Sidekiq::Worker + include ClusterQueue + include ClusterApplications + + def perform(app_name, app_id) + find_application(app_name, app_id) do |app| + Clusters::Applications::InstallService.new(app).execute + end + end +end diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb new file mode 100644 index 00000000000..4bb8c293e5d --- /dev/null +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -0,0 +1,14 @@ +class ClusterWaitForAppInstallationWorker + include Sidekiq::Worker + include ClusterQueue + include ClusterApplications + + INTERVAL = 10.seconds + TIMEOUT = 20.minutes + + def perform(app_name, app_id) + find_application(app_name, app_id) do |app| + Clusters::Applications::CheckInstallationProgressService.new(app).execute + end + end +end diff --git a/app/workers/concerns/cluster_applications.rb b/app/workers/concerns/cluster_applications.rb new file mode 100644 index 00000000000..24ecaa0b52f --- /dev/null +++ b/app/workers/concerns/cluster_applications.rb @@ -0,0 +1,9 @@ +module ClusterApplications + extend ActiveSupport::Concern + + included do + def find_application(app_name, id, &blk) + Clusters::Cluster::APPLICATIONS[app_name].find(id).try(&blk) + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/notify_upon_death.rb b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb new file mode 100644 index 00000000000..3d7120665b6 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/notify_upon_death.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # NotifyUponDeath can be included into a GitHub worker class if it should + # notify any JobWaiter instances upon being moved to the Sidekiq dead queue. + # + # Note that this will only notify the waiter upon graceful termination, a + # SIGKILL will still result in the waiter _not_ being notified. + # + # Workers including this module must have jobs passed where the last + # argument is the key to notify, as a String. + module NotifyUponDeath + extend ActiveSupport::Concern + + included do + # If a job is being exhausted we still want to notify the + # AdvanceStageWorker. This prevents the entire import from getting stuck + # just because 1 job threw too many errors. + sidekiq_retries_exhausted do |job| + args = job['args'] + jid = job['jid'] + + if args.length == 3 && (key = args.last) && key.is_a?(String) + JobWaiter.notify(key, jid) + end + end + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb new file mode 100644 index 00000000000..67e36c811de --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # ObjectImporter defines the base behaviour for every Sidekiq worker that + # imports a single resource such as a note or pull request. + module ObjectImporter + extend ActiveSupport::Concern + + included do + include Sidekiq::Worker + include GithubImport::Queue + include ReschedulingMethods + include NotifyUponDeath + end + + # project - An instance of `Project` to import the data into. + # client - An instance of `Gitlab::GithubImport::Client` + # hash - A Hash containing the details of the object to import. + def import(project, client, hash) + object = representation_class.from_json_hash(hash) + + importer_class.new(object, project, client).execute + + counter.increment(project: project.path_with_namespace) + end + + def counter + @counter ||= Gitlab::Metrics.counter(counter_name, counter_description) + end + + # Returns the representation class to use for the object. This class must + # define the class method `from_json_hash`. + def representation_class + raise NotImplementedError + end + + # Returns the class to use for importing the object. + def importer_class + raise NotImplementedError + end + + # Returns the name (as a Symbol) of the Prometheus counter. + def counter_name + raise NotImplementedError + end + + # Returns the description (as a String) of the Prometheus counter. + def counter_description + raise NotImplementedError + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/queue.rb b/app/workers/concerns/gitlab/github_import/queue.rb new file mode 100644 index 00000000000..a2bee361b86 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/queue.rb @@ -0,0 +1,16 @@ +module Gitlab + module GithubImport + module Queue + extend ActiveSupport::Concern + + included do + # If a job produces an error it may block a stage from advancing + # forever. To prevent this from happening we prevent jobs from going to + # the dead queue. This does mean some resources may not be imported, but + # this is better than a project being stuck in the "import" state + # forever. + sidekiq_options queue: 'github_importer', dead: false, retry: 5 + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb new file mode 100644 index 00000000000..692ca6b7f42 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Module that provides methods shared by the various workers used for + # importing GitHub projects. + module ReschedulingMethods + # project_id - The ID of the GitLab project to import the note into. + # hash - A Hash containing the details of the GitHub object to imoprt. + # notify_key - The Redis key to notify upon completion, if any. + def perform(project_id, hash, notify_key = nil) + project = Project.find_by(id: project_id) + + return notify_waiter(notify_key) unless project + + client = GithubImport.new_client_for(project, parallel: true) + + if try_import(project, client, hash) + notify_waiter(notify_key) + else + # In the event of hitting the rate limit we want to reschedule the job + # so its retried after our rate limit has been reset. + self.class + .perform_in(client.rate_limit_resets_in, project.id, hash, notify_key) + end + end + + def try_import(*args) + import(*args) + true + rescue RateLimitError + false + end + + def notify_waiter(key = nil) + JobWaiter.notify(key, jid) if key + end + end + end +end diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb new file mode 100644 index 00000000000..147c8c8d683 --- /dev/null +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module StageMethods + # project_id - The ID of the GitLab project to import the data into. + def perform(project_id) + return unless (project = find_project(project_id)) + + client = GithubImport.new_client_for(project) + + try_import(client, project) + end + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def try_import(client, project) + import(client, project) + rescue RateLimitError + self.class.perform_in(client.rate_limit_resets_in, project.id) + end + + def find_project(id) + # If the project has been marked as failed we want to bail out + # automatically. + Project.import_started.find_by(id: id) + end + end + end +end diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb new file mode 100644 index 00000000000..877f88c043f --- /dev/null +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # AdvanceStageWorker is a worker used by the GitHub importer to wait for a + # number of jobs to complete, without blocking a thread. Once all jobs have + # been completed this worker will advance the import process to the next + # stage. + class AdvanceStageWorker + include Sidekiq::Worker + + sidekiq_options queue: 'github_importer_advance_stage', dead: false + + INTERVAL = 30.seconds.to_i + + # The number of seconds to wait (while blocking the thread) before + # continueing to the next waiter. + BLOCKING_WAIT_TIME = 5 + + # The known importer stages and their corresponding Sidekiq workers. + STAGES = { + issues_and_diff_notes: Stage::ImportIssuesAndDiffNotesWorker, + notes: Stage::ImportNotesWorker, + finish: Stage::FinishImportWorker + }.freeze + + # project_id - The ID of the project being imported. + # waiters - A Hash mapping Gitlab::JobWaiter keys to the number of + # remaining jobs. + # next_stage - The name of the next stage to start when all jobs have been + # completed. + def perform(project_id, waiters, next_stage) + return unless (project = find_project(project_id)) + + new_waiters = wait_for_jobs(waiters) + + if new_waiters.empty? + # We refresh the import JID here so workers importing individual + # resources (e.g. notes) don't have to do this all the time, reducing + # the pressure on Redis. We _only_ do this once all jobs are done so + # we don't get stuck forever if one or more jobs failed to notify the + # JobWaiter. + project.refresh_import_jid_expiration + + STAGES.fetch(next_stage.to_sym).perform_async(project_id) + else + self.class.perform_in(INTERVAL, project_id, new_waiters, next_stage) + end + end + + def wait_for_jobs(waiters) + waiters.each_with_object({}) do |(key, remaining), new_waiters| + waiter = JobWaiter.new(remaining, key) + + # We wait for a brief moment of time so we don't reschedule if we can + # complete the work fast enough. + waiter.wait(BLOCKING_WAIT_TIME) + + next unless waiter.jobs_remaining.positive? + + new_waiters[waiter.key] = waiter.jobs_remaining + end + end + + def find_project(id) + # We only care about the import JID so we can refresh it. We also only + # want the project if it hasn't been marked as failed yet. It's possible + # the import gets marked as stuck when jobs of the current stage failed + # somehow. + Project.select(:import_jid).import_started.find_by(id: id) + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_diff_note_worker.rb b/app/workers/gitlab/github_import/import_diff_note_worker.rb new file mode 100644 index 00000000000..ef2a74c51c5 --- /dev/null +++ b/app/workers/gitlab/github_import/import_diff_note_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportDiffNoteWorker + include ObjectImporter + + def representation_class + Representation::DiffNote + end + + def importer_class + Importer::DiffNoteImporter + end + + def counter_name + :github_importer_imported_diff_notes + end + + def counter_description + 'The number of imported GitHub pull request review comments' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_issue_worker.rb b/app/workers/gitlab/github_import/import_issue_worker.rb new file mode 100644 index 00000000000..1b081ae5966 --- /dev/null +++ b/app/workers/gitlab/github_import/import_issue_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportIssueWorker + include ObjectImporter + + def representation_class + Representation::Issue + end + + def importer_class + Importer::IssueAndLabelLinksImporter + end + + def counter_name + :github_importer_imported_issues + end + + def counter_description + 'The number of imported GitHub issues' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_note_worker.rb b/app/workers/gitlab/github_import/import_note_worker.rb new file mode 100644 index 00000000000..d2b4c36a5b9 --- /dev/null +++ b/app/workers/gitlab/github_import/import_note_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportNoteWorker + include ObjectImporter + + def representation_class + Representation::Note + end + + def importer_class + Importer::NoteImporter + end + + def counter_name + :github_importer_imported_notes + end + + def counter_description + 'The number of imported GitHub comments' + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb new file mode 100644 index 00000000000..62a6da152a3 --- /dev/null +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ImportPullRequestWorker + include ObjectImporter + + def representation_class + Representation::PullRequest + end + + def importer_class + Importer::PullRequestImporter + end + + def counter_name + :github_importer_imported_pull_requests + end + + def counter_description + 'The number of imported GitHub pull requests' + end + end + end +end diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb new file mode 100644 index 00000000000..45a38927225 --- /dev/null +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class RefreshImportJidWorker + include Sidekiq::Worker + include GithubImport::Queue + + # The interval to schedule new instances of this job at. + INTERVAL = 1.minute.to_i + + def self.perform_in_the_future(*args) + perform_in(INTERVAL, *args) + end + + # project_id - The ID of the project that is being imported. + # check_job_id - The ID of the job for which to check the status. + def perform(project_id, check_job_id) + return unless (project = find_project(project_id)) + + if SidekiqStatus.running?(check_job_id) + # As long as the repository is being cloned we want to keep refreshing + # the import JID status. + project.refresh_import_jid_expiration + self.class.perform_in_the_future(project_id, check_job_id) + end + + # If the job is no longer running there's nothing else we need to do. If + # the clone job completed successfully it will have scheduled the next + # stage, if it died there's nothing we can do anyway. + end + + def find_project(id) + Project.select(:import_jid).import_started.find_by(id: id) + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb new file mode 100644 index 00000000000..1a09497780a --- /dev/null +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class FinishImportWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # project - An instance of Project. + def import(_, project) + project.after_import + report_import_time(project) + end + + def report_import_time(project) + duration = Time.zone.now - project.created_at + path = project.path_with_namespace + + histogram.observe({ project: path }, duration) + counter.increment + + logger.info("GitHub importer finished for #{path} in #{duration.round(2)} seconds") + end + + def histogram + @histogram ||= Gitlab::Metrics.histogram( + :github_importer_total_duration_seconds, + 'Total time spent importing GitHub projects, in seconds' + ) + end + + def counter + @counter ||= Gitlab::Metrics.counter( + :github_importer_imported_projects, + 'The number of imported GitHub projects' + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb new file mode 100644 index 00000000000..f8a3684c6ba --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportBaseDataWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # These importers are fast enough that we can just run them in the same + # thread. + IMPORTERS = [ + Importer::LabelsImporter, + Importer::MilestonesImporter, + Importer::ReleasesImporter + ].freeze + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + project.refresh_import_jid_expiration + + ImportPullRequestsWorker.perform_async(project.id) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb new file mode 100644 index 00000000000..e110b7c1c36 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportIssuesAndDiffNotesWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # The importers to run in this stage. Issues can't be imported earlier + # on as we also use these to enrich pull requests with assigned labels. + IMPORTERS = [ + Importer::IssuesImporter, + Importer::DiffNotesImporter + ].freeze + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiters = IMPORTERS.each_with_object({}) do |klass, hash| + waiter = klass.new(project, client).execute + hash[waiter.key] = waiter.jobs_remaining + end + + AdvanceStageWorker.perform_async(project.id, waiters, :notes) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb new file mode 100644 index 00000000000..9810ed25cf9 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportNotesWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = Importer::NotesImporter + .new(project, client) + .execute + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :finish + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb new file mode 100644 index 00000000000..c531f26e897 --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportPullRequestsWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + waiter = Importer::PullRequestsImporter + .new(project, client) + .execute + + project.refresh_import_jid_expiration + + AdvanceStageWorker.perform_async( + project.id, + { waiter.key => waiter.jobs_remaining }, + :issues_and_diff_notes + ) + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb new file mode 100644 index 00000000000..aa5762e773d --- /dev/null +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Stage + class ImportRepositoryWorker + include Sidekiq::Worker + include GithubImport::Queue + include StageMethods + + # client - An instance of Gitlab::GithubImport::Client. + # project - An instance of Project. + def import(client, project) + # In extreme cases it's possible for a clone to take more than the + # import job expiration time. To work around this we schedule a + # separate job that will periodically run and refresh the import + # expiration time. + RefreshImportJidWorker.perform_in_the_future(project.id, jid) + + importer = Importer::RepositoryImporter.new(project, client) + + return unless importer.execute + + counter.increment + + ImportBaseDataWorker.perform_async(project.id) + end + + def counter + Gitlab::Metrics.counter( + :github_importer_imported_repositories, + 'The number of imported GitHub repositories' + ) + end + end + end + end +end diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index d7c0043d3b6..4e90b137b26 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -17,11 +17,16 @@ class RepositoryImportWorker import_url: project.import_url, path: project.full_path) - result = Projects::ImportService.new(project, project.creator).execute + service = Projects::ImportService.new(project, project.creator) + result = service.execute + + # Some importers may perform their work asynchronously. In this case it's up + # to those importers to mark the import process as complete. + return if service.async? + raise ImportError, result[:message] if result[:status] == :error - project.repository.after_import - project.import_finish + project.after_import rescue ImportError => ex fail_import(project, ex.message) raise diff --git a/changelogs/unreleased/36629-35958-add-cluster-application-section.yml b/changelogs/unreleased/36629-35958-add-cluster-application-section.yml new file mode 100644 index 00000000000..0afa53e8642 --- /dev/null +++ b/changelogs/unreleased/36629-35958-add-cluster-application-section.yml @@ -0,0 +1,6 @@ +--- +title: Add applications section to GKE clusters page to easily install Helm Tiller, + Ingress +merge_request: +author: +type: added diff --git a/changelogs/unreleased/37824-many-branches-lock-server.yml b/changelogs/unreleased/37824-many-branches-lock-server.yml new file mode 100644 index 00000000000..f75f79ec4a0 --- /dev/null +++ b/changelogs/unreleased/37824-many-branches-lock-server.yml @@ -0,0 +1,6 @@ +--- +title: While displaying a commit, do not show list of related branches if there are + thousands of branches +merge_request: 14812 +author: +type: other diff --git a/changelogs/unreleased/38394-smarter-interval.yml b/changelogs/unreleased/38394-smarter-interval.yml new file mode 100644 index 00000000000..ead8c3eca5a --- /dev/null +++ b/changelogs/unreleased/38394-smarter-interval.yml @@ -0,0 +1,5 @@ +--- +title: Update Merge Request polling so there is only one request at a time +merge_request: 15032 +author: +type: fixed diff --git a/changelogs/unreleased/38395-mr-widget-ci.yml b/changelogs/unreleased/38395-mr-widget-ci.yml new file mode 100644 index 00000000000..5109f1bec44 --- /dev/null +++ b/changelogs/unreleased/38395-mr-widget-ci.yml @@ -0,0 +1,6 @@ +--- +title: Moves mini graph of pipeline to the end of sentence in MR widget. Cleans HTML + and tests +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/39878-commit-pipeline-reads-wrong-key.yml b/changelogs/unreleased/39878-commit-pipeline-reads-wrong-key.yml new file mode 100644 index 00000000000..b24edfe0cb9 --- /dev/null +++ b/changelogs/unreleased/39878-commit-pipeline-reads-wrong-key.yml @@ -0,0 +1,5 @@ +--- +title: Fix commit pipeline showing wrong status +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/add-ingress-to-cluster-applications.yml b/changelogs/unreleased/add-ingress-to-cluster-applications.yml new file mode 100644 index 00000000000..0064e8672f8 --- /dev/null +++ b/changelogs/unreleased/add-ingress-to-cluster-applications.yml @@ -0,0 +1,5 @@ +--- +title: Add Ingress to available Cluster applications +merge_request: +author: +type: added diff --git a/changelogs/unreleased/add-typescript.yml b/changelogs/unreleased/add-typescript.yml new file mode 100644 index 00000000000..a048f240855 --- /dev/null +++ b/changelogs/unreleased/add-typescript.yml @@ -0,0 +1,5 @@ +--- +title: Adds typescript support +merge_request: +author: +type: added diff --git a/changelogs/unreleased/animate-auto-devops.yml b/changelogs/unreleased/animate-auto-devops.yml deleted file mode 100644 index c572dbdd093..00000000000 --- a/changelogs/unreleased/animate-auto-devops.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Animate auto devops graphic -merge_request: -author: -type: other diff --git a/changelogs/unreleased/bvl-free-paths.yml b/changelogs/unreleased/bvl-free-paths.yml new file mode 100644 index 00000000000..f15459cc788 --- /dev/null +++ b/changelogs/unreleased/bvl-free-paths.yml @@ -0,0 +1,5 @@ +--- +title: Free up some reserved group names +merge_request: 15052 +author: +type: other diff --git a/changelogs/unreleased/feature-change-signout-route.yml b/changelogs/unreleased/feature-change-signout-route.yml new file mode 100644 index 00000000000..bccb85b3eaf --- /dev/null +++ b/changelogs/unreleased/feature-change-signout-route.yml @@ -0,0 +1,5 @@ +--- +title: Change 'Sign Out' route from a DELETE to a GET +merge_request: 39708 +author: Joe Marty +type: changed diff --git a/changelogs/unreleased/feature-hashed-storage-repo-import.yml b/changelogs/unreleased/feature-hashed-storage-repo-import.yml new file mode 100644 index 00000000000..73c16a99053 --- /dev/null +++ b/changelogs/unreleased/feature-hashed-storage-repo-import.yml @@ -0,0 +1,5 @@ +--- +title: Improve GitLab Import rake task to work with Hashed Storage and Subgroups +merge_request: +author: +type: changed diff --git a/changelogs/unreleased/fix-import-export-arguments.yml b/changelogs/unreleased/fix-import-export-arguments.yml new file mode 100644 index 00000000000..eee87e313ea --- /dev/null +++ b/changelogs/unreleased/fix-import-export-arguments.yml @@ -0,0 +1,5 @@ +--- +title: Fix arguments Import/Export error importing project merge requests +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/fix-mysql-grant-check.yml b/changelogs/unreleased/fix-mysql-grant-check.yml new file mode 100644 index 00000000000..a1c1aa67d79 --- /dev/null +++ b/changelogs/unreleased/fix-mysql-grant-check.yml @@ -0,0 +1,5 @@ +--- +title: Fix TRIGGER checks for MySQL +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/tc-delete-merged-protected-tags-fix.yml b/changelogs/unreleased/tc-delete-merged-protected-tags-fix.yml new file mode 100644 index 00000000000..5d5c39108b0 --- /dev/null +++ b/changelogs/unreleased/tc-delete-merged-protected-tags-fix.yml @@ -0,0 +1,5 @@ +--- +title: When deleting merged branches, ignore protected tags +merge_request: 15252 +author: +type: fixed diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index c6ec0aeda7b..958859be6cf 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -195,7 +195,7 @@ Devise.setup do |config| config.navigational_formats = [:"*/*", "*/*", :html, :zip] # The default HTTP method used to sign out a resource. Default is :delete. - config.sign_out_via = :delete + config.sign_out_via = :get # ==> OmniAuth # To configure a new OmniAuth provider copy and edit omniauth.rb.sample diff --git a/config/karma.config.js b/config/karma.config.js index e459f5cdac3..9f018d14b8f 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -67,14 +67,5 @@ module.exports = function(config) { karmaConfig.customLaunchers.ChromeHeadlessCustom.flags.push('--enable-logging', '--v=1'); } - if (process.env.DEBUG) { - karmaConfig.logLevel = config.LOG_DEBUG; - process.env.CHROME_LOG_FILE = process.env.CHROME_LOG_FILE || 'chrome_debug.log'; - } - - if (process.env.CHROME_LOG_FILE) { - karmaConfig.customLaunchers.ChromeHeadlessCustom.flags.push('--enable-logging', '--v=1'); - } - config.set(karmaConfig); }; diff --git a/config/routes/group.rb b/config/routes/group.rb index f4d520a2518..db99e10bb9a 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -8,24 +8,33 @@ constraints(GroupUrlConstrainer.new) do scope(path: 'groups/*id', controller: :groups, constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do - get :edit, as: :edit_group - get :issues, as: :issues_group - get :merge_requests, as: :merge_requests_group - get :projects, as: :projects_group - get :activity, as: :activity_group + scope(path: '-') do + get :edit, as: :edit_group + get :issues, as: :issues_group + get :merge_requests, as: :merge_requests_group + get :projects, as: :projects_group + get :activity, as: :activity_group + end + get '/', action: :show, as: :group_canonical end - scope(path: 'groups/*group_id', + scope(path: 'groups/*group_id/-', module: :groups, as: :group, constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do - resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do - post :resend_invite, on: :member - delete :leave, on: :collection + namespace :settings do + resource :ci_cd, only: [:show], controller: 'ci_cd' + end + + resources :variables, only: [:index, :show, :update, :create, :destroy] + + resources :children, only: [:index] + + resources :labels, except: [:show] do + post :toggle_subscription, on: :member end - resource :avatar, only: [:destroy] resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do member do get :merge_requests @@ -34,18 +43,11 @@ constraints(GroupUrlConstrainer.new) do end end - resources :labels, except: [:show] do - post :toggle_subscription, on: :member - end - - scope path: '-' do - namespace :settings do - resource :ci_cd, only: [:show], controller: 'ci_cd' - end - - resources :variables, only: [:index, :show, :update, :create, :destroy] + resource :avatar, only: [:destroy] - resources :children, only: [:index] + resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do + post :resend_invite, on: :member + delete :leave, on: :collection end end @@ -58,4 +60,12 @@ constraints(GroupUrlConstrainer.new) do put '/', action: :update delete '/', action: :destroy end + + # Legacy paths should be defined last, so they would be ignored if routes with + # one of the previously reserved words exist. + scope(path: 'groups/*group_id') do + Gitlab::Routing.redirect_legacy_paths(self, :labels, :milestones, :group_members, + :edit, :issues, :merge_requests, :projects, + :activity) + end end diff --git a/config/routes/project.rb b/config/routes/project.rb index c52dd542c90..bdafaba3ab3 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -191,6 +191,10 @@ constraints(ProjectUrlConstrainer.new) do member do get :status, format: :json + + scope :applications do + post '/:application', to: 'clusters/applications#create', as: :install_applications + end end end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index e2bb766ee47..a8b918177de 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -40,6 +40,8 @@ - [upload_checksum, 1] - [repository_fork, 1] - [repository_import, 1] + - [github_importer, 1] + - [github_importer_advance_stage, 1] - [project_service, 1] - [delete_user, 1] - [delete_merged_branches, 1] diff --git a/config/webpack.config.js b/config/webpack.config.js index f7a7182a627..67d7cae3ccf 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -108,6 +108,10 @@ var config = { loader: 'vue-loader', }, { + test: /\.ts$/, + loader: 'ts-loader', + }, + { test: /\.svg$/, loader: 'raw-loader', }, @@ -252,7 +256,7 @@ var config = { ], resolve: { - extensions: ['.js'], + extensions: ['.js', '.ts'], alias: { '~': path.join(ROOT_PATH, 'app/assets/javascripts'), 'emojis': path.join(ROOT_PATH, 'fixtures/emojis'), diff --git a/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb b/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb new file mode 100644 index 00000000000..a2ce37127ea --- /dev/null +++ b/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb @@ -0,0 +1,18 @@ +class CreateClustersKubernetesHelmApps < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :clusters_applications_helm do |t| + t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade } + + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :updated_at, null: false + + t.integer :status, null: false + t.string :version, null: false + t.text :status_reason + end + end +end diff --git a/db/migrate/20171106101200_create_clusters_kubernetes_ingress_apps.rb b/db/migrate/20171106101200_create_clusters_kubernetes_ingress_apps.rb new file mode 100644 index 00000000000..21f48b1d1b4 --- /dev/null +++ b/db/migrate/20171106101200_create_clusters_kubernetes_ingress_apps.rb @@ -0,0 +1,21 @@ +class CreateClustersKubernetesIngressApps < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :clusters_applications_ingress do |t| + t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade } + + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :updated_at, null: false + + t.integer :status, null: false + t.integer :ingress_type, null: false + + t.string :version, null: false + t.string :cluster_ip + t.text :status_reason + end + end +end diff --git a/db/schema.rb b/db/schema.rb index c4c43235f02..c60cb729b75 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20171101134435) do +ActiveRecord::Schema.define(version: 20171106101200) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -519,6 +519,26 @@ ActiveRecord::Schema.define(version: 20171101134435) do add_index "clusters", ["enabled"], name: "index_clusters_on_enabled", using: :btree add_index "clusters", ["user_id"], name: "index_clusters_on_user_id", using: :btree + create_table "clusters_applications_helm", force: :cascade do |t| + t.integer "cluster_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.integer "status", null: false + t.string "version", null: false + t.text "status_reason" + end + + create_table "clusters_applications_ingress", force: :cascade do |t| + t.integer "cluster_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.integer "status", null: false + t.integer "ingress_type", null: false + t.string "version", null: false + t.string "cluster_ip" + t.text "status_reason" + end + create_table "container_repositories", force: :cascade do |t| t.integer "project_id", null: false t.string "name", null: false @@ -1893,6 +1913,7 @@ ActiveRecord::Schema.define(version: 20171101134435) do add_foreign_key "cluster_projects", "projects", on_delete: :cascade add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade add_foreign_key "clusters", "users", on_delete: :nullify + add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade add_foreign_key "container_repositories", "projects" add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade diff --git a/doc/README.md b/doc/README.md index 7c33a708dc7..d4119d35162 100644 --- a/doc/README.md +++ b/doc/README.md @@ -171,6 +171,7 @@ have access to GitLab administration tools and settings. - [GitLab performance monitoring with InfluxDB](administration/monitoring/performance/introduction.md): Configure GitLab and InfluxDB for measuring performance metrics. - [GitLab performance monitoring with Prometheus](administration/monitoring/prometheus/index.md): Configure GitLab and Prometheus for measuring performance metrics. - [Monitoring uptime](user/admin_area/monitoring/health_check.md): Check the server status using the health check endpoint. +- [Monitoring GitHub imports](administration/monitoring/github_imports.md) ### Performance diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md new file mode 100644 index 00000000000..5592e0a9e9a --- /dev/null +++ b/doc/administration/monitoring/github_imports.md @@ -0,0 +1,101 @@ +# Monitoring GitHub imports + +>**Note:** +Available since [GitLab 10.2][14731]. + +The GitHub importer exposes various Prometheus metrics that you can use to +monitor the health and progress of the importer. + +## Import Duration Times + +| Name | Type | +|------------------------------------------|-----------| +| `github_importer_total_duration_seconds` | histogram | + +This metric tracks the total time spent (in seconds) importing a project (from +project creation until the import process finishes), for every imported project. + +The name of the project is stored in the `project` label in the format +`namespace/name` (e.g. `gitlab-org/gitlab-ce`). + +## Number of imported projects + +| Name | Type | +|-------------------------------------|---------| +| `github_importer_imported_projects` | counter | + +This metric tracks the total number of projects imported over time. This metric +does not expose any labels. + +## Number of GitHub API calls + +| Name | Type | +|---------------------------------|---------| +| `github_importer_request_count` | counter | + +This metric tracks the total number of GitHub API calls performed over time, for +all projects. This metric does not expose any labels. + +## Rate limit errors + +| Name | Type | +|-----------------------------------|---------| +| `github_importer_rate_limit_hits` | counter | + +This metric tracks the number of times we hit the GitHub rate limit, for all +projects. This metric does not expose any labels. + +## Number of imported issues + +| Name | Type | +|-----------------------------------|---------| +| `github_importer_imported_issues` | counter | + +This metric tracks the number of imported issues across all projects. + +The name of the project is stored in the `project` label in the format +`namespace/name` (e.g. `gitlab-org/gitlab-ce`). + +## Number of imported pull requests + +| Name | Type | +|------------------------------------------|---------| +| `github_importer_imported_pull_requests` | counter | + +This metric tracks the number of imported pull requests across all projects. + +The name of the project is stored in the `project` label in the format +`namespace/name` (e.g. `gitlab-org/gitlab-ce`). + +## Number of imported comments + +| Name | Type | +|----------------------------------|---------| +| `github_importer_imported_notes` | counter | + +This metric tracks the number of imported comments across all projects. + +The name of the project is stored in the `project` label in the format +`namespace/name` (e.g. `gitlab-org/gitlab-ce`). + +## Number of imported pull request review comments + +| Name | Type | +|---------------------------------------|---------| +| `github_importer_imported_diff_notes` | counter | + +This metric tracks the number of imported comments across all projects. + +The name of the project is stored in the `project` label in the format +`namespace/name` (e.g. `gitlab-org/gitlab-ce`). + +## Number of imported repositories + +| Name | Type | +|-----------------------------------------|---------| +| `github_importer_imported_repositories` | counter | + +This metric tracks the number of imported repositories across all projects. This +metric does not expose any labels. + +[14731]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14731 diff --git a/doc/development/README.md b/doc/development/README.md index 0cafc112b6b..5690ae68e00 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -37,6 +37,7 @@ comments: false - [Gotchas](gotchas.md) to avoid - [Issue and merge requests state models](object_state_models.md) - [How to dump production data to staging](db_dump.md) +- [Working with the GitHub importer](github_importer.md) ## Performance guides diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md new file mode 100644 index 00000000000..0d558583bb8 --- /dev/null +++ b/doc/development/github_importer.md @@ -0,0 +1,209 @@ +# Working with the GitHub importer + +In GitLab 10.2 a new version of the GitHub importer was introduced. This new +importer performs its work in parallel using Sidekiq, greatly reducing the time +necessary to import GitHub projects into a GitLab instance. + +The GitHub importer offers two different types of importers: a sequential +importer and a parallel importer. The Rake task `import:github` uses the +sequential importer, while everything else uses the parallel importer. The +difference between these two importers is quite simple: the sequential importer +does all work in a single thread, making it more useful for debugging purposes +or Rake tasks. The parallel importer on the other hand uses Sidekiq. + +## Requirements + +* GitLab CE 10.2.0 or newer. +* Sidekiq workers that process the `github_importer` and + `github_importer_advance_stage` queues (this is enabled by default). +* Octokit (used for interacting with the GitHub API) + +## Code structure + +The importer's codebase is broken up into the following directories: + +* `lib/gitlab/github_import`: this directory contains most of the code such as + the classes used for importing resources. +* `app/workers/gitlab/github_import`: this directory contains the Sidekiq + workers. +* `app/workers/concerns/gitlab/github_import`: this directory contains a few + modules reused by the various Sidekiq workers. + +## Architecture overview + +When a GitHub project is imported we schedule and execute a job for the +`RepositoryImportworker` worker as all other importers. However, unlike other +importers we don't immediately perform the work necessary. Instead work is +divided into separate stages, with each stage consisting out of a set of Sidekiq +jobs that are executed. Between every stage a job is scheduled that periodically +checks if all work of the current stage is completed, advancing the import +process to the next stage when this is the case. The worker handling this is +called `Gitlab::GithubImport::AdvanceStageWorker`. + +## Stages + +### 1. RepositoryImportWorker + +This worker will kick off the import process by simply scheduling a job for the +next worker. + +### 2. Stage::ImportRepositoryWorker + +This worker will import the repository and wiki, scheduling the next stage when +done. + +### 3. Stage::ImportBaseDataWorker + +This worker will import base data such as labels, milestones, and releases. This +work is done in a single thread since it can be performed fast enough that we +don't need to perform this work in parallel. + +### 4. Stage::ImportPullRequestsWorker + +This worker will import all pull requests. For every pull request a job for the +`Gitlab::GithubImport::ImportPullRequestWorker` worker is scheduled. + +### 5. Stage::ImportIssuesAndDiffNotesWorker + +This worker will import all issues and pull request comments. For every issue we +schedule a job for the `Gitlab::GithubImport::ImportIssueWorker` worker. For +pull request comments we instead schedule jobs for the +`Gitlab::GithubImport::DiffNoteImporter` worker. + +This worker processes both issues and diff notes in parallel so we don't need to +schedule a separate stage and wait for the previous one to complete. + +Issues are imported separately from pull requests because only the "issues" API +includes labels for both issue and pull requests. Importing issues and setting +label links in the same worker removes the need for performing a separate crawl +through the API data, reducing the number of API calls necessary to import a +project. + +### 6. Stage::ImportNotesWorker + +This worker imports regular comments for both issues and pull requests. For +every comment we schedule a job for the +`Gitlab::GithubImport::ImportNoteWorker` worker. + +Regular comments have to be imported at the end since the GitHub API used +returns comments for both issues and pull requests. This means we have to wait +for all issues and pull requests to be imported before we can import regular +comments. + +### 7. Stage::FinishImportWorker + +This worker will wrap up the import process by performing some housekeeping +(such as flushing any caches) and by marking the import as completed. + +## Advancing stages + +Advancing stages is done in one of two ways: + +1. Scheduling the worker for the next stage directly. +2. Scheduling a job for `Gitlab::GithubImport::AdvanceStageWorker` which will + advance the stage when all work of the current stage has been completed. + +The first approach should only be used by workers that perform all their work in +a single thread, while `AdvanceStageWorker` should be used for everything else. + +The way `AdvanceStageWorker` works is fairly simple. When scheduling a job it +will be given a project ID, a list of Redis keys, and the name of the next +stage. The Redis keys (produced by `Gitlab::JobWaiter`) are used to check if the +currently running stage has been completed or not. If the stage has not yet been +completed `AdvanceStageWorker` will reschedule itself. Once a stage finishes +`AdvanceStageworker` will refresh the import JID (more on this below) and +schedule the worker of the next stage. + +To reduce the number of `AdvanceStageWorker` jobs scheduled this worker will +briefly wait for jobs to complete before deciding what the next action should +be. For small projects this may slow down the import process a bit, but it will +also reduce pressure on the system as a whole. + +## Refreshing import JIDs + +GitLab includes a worker called `StuckImportJobsWorker` that will periodically +run and mark project imports as failed if they have been running for more than +15 hours. For GitHub projects this poses a bit of a problem: importing large +projects could take several hours depending on how often we hit the GitHub rate +limit (more on this below), but we don't want `StuckImportJobsWorker` to mark +our import as failed because of this. + +To prevent this from happening we periodically refresh the expiration time of +the import process. This works by storing the JID of the import job in the +database, then refreshing this JID's TTL at various stages throughout the import +process. This is done by calling `Project#refresh_import_jid_expiration`. By +refreshing this TTL we can ensure our import does not get marked as failed so +long we're still performing work. + +## GitHub rate limit + +GitHub has a rate limit of 5 000 API calls per hour. The number of requests +necessary to import a project is largely dominated by the number of unique users +involved in a project (e.g. issue authors). Other data such as issue pages +and comments typically only requires a few dozen requests to import. This is +because we need the Email address of users in order to map them to GitLab users. + +We handle this by doing the following: + +1. Once we hit the rate limit all jobs will automatically reschedule themselves + in such a way that they are not executed until the rate limit has been reset. +2. We cache the mapping of GitHub users to GitLab users in Redis. + +More information on user caching can be found below. + +## Caching user lookups + +When mapping GitHub users to GitLab users we need to (in the worst case) +perform: + +1. One API call to get the user's Email address. +2. Two database queries to see if a corresponding GitLab user exists. One query + will try to find the user based on the GitHub user ID, while the second query + is used to find the user using their GitHub Email address. + +Because this process is quite expensive we cache the result of these lookups in +Redis. For every user looked up we store three keys: + +1. A Redis key mapping GitHub usernames to their Email addresses. +2. A Redis key mapping a GitHub Email addresses to a GitLab user ID. +3. A Redis key mapping a GitHub user ID to GitLab user ID. + +There are two types of lookups we cache: + +1. A positive lookup, meaning we found a GitLab user ID. +2. A negative lookup, meaning we didn't find a GitLab user ID. Caching this + prevents us from performing the same work for users that we know don't exist + in our GitLab database. + +The expiration time of these keys is 24 hours. When retrieving the cache of a +positive lookups we refresh the TTL automatically. The TTL of false lookups is +never refreshed. + +Because of this caching layer it's possible newly registered GitLab accounts +won't be linked to their corresponding GitHub accounts. This however will sort +itself out once the cached keys expire. + +The user cache lookup is shared across projects. This means that the more +projects get imported the fewer GitHub API calls will be needed. + +The code for this resides in: + +* `lib/gitlab/github_import/user_finder.rb` +* `lib/gitlab/github_import/caching.rb` + +## Mapping labels and milestones + +To reduce pressure on the database we do not query it when setting labels and +milestones on issues and merge requests. Instead we cache this data when we +import labels and milestones, then we reuse this cache when assigning them to +issues/merge requests. Similar to the user lookups these cache keys are expired +automatically after 24 hours of not being used. + +Unlike the user lookup caches these label and milestone caches are scoped to the +project that is being imported. + +The code for this resides in: + +* `lib/gitlab/github_import/label_finder.rb` +* `lib/gitlab/github_import/milestone_finder.rb` +* `lib/gitlab/github_import/caching.rb` diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 7c38260406d..4b65a0f4a35 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -110,7 +110,7 @@ You can mark that content for translation with: In JavaScript we added the `__()` (double underscore parenthesis) function for translations. -### Updating the PO files with the new content +## Updating the PO files with the new content Now that the new content is marked for translation, we need to update the PO files with the following command: @@ -119,23 +119,20 @@ files with the following command: bundle exec rake gettext:find ``` -This command will update the `locale/**/gitlab.edit.po` file with the -new content that the parser has found. +This command will update the `locale/gitlab.pot` file with the newly externalized +strings and remove any strings that aren't used anymore. You should check this +file in. Once the changes are on master, they will be picked up by +[Crowdin](http://translate.gitlab.com) and be presented for translation. -New translations will be added with their default content and will be marked -fuzzy. To use the translation, look for the `#, fuzzy` mention in `gitlab.edit.po` -and remove it. +The command also updates the translation files for each language: `locale/*/gitlab.po` +These changes can be discarded, the languange files will be updated by Crowdin +automatically. -We need to make sure we remove the `fuzzy` translations before generating the -`locale/**/gitlab.po` file. When they aren't removed, the resulting `.po` will -be treated as a binary file which could overwrite translations that were merged -before the new translations. +Discard all of them at once like this: -When we are just preparing a page to be translated, but not actually adding any -translations. There's no need to generate `.po` files. - -Translations that aren't used in the source code anymore will be marked with -`~#`; these can be removed to keep our translation files clutter-free. +```sh +git checkout locale/*/gitlab.po +``` ### Validating PO files diff --git a/doc/development/ux_guide/components.md b/doc/development/ux_guide/components.md index fa31c496b30..16a811dbc74 100644 --- a/doc/development/ux_guide/components.md +++ b/doc/development/ux_guide/components.md @@ -10,6 +10,7 @@ * [Tables](#tables) * [Blocks](#blocks) * [Panels](#panels) +* [Dialog modals](#dialog-modals) * [Alerts](#alerts) * [Forms](#forms) * [Search box](#search-box) @@ -254,6 +255,38 @@ Skeleton loading can replace any existing UI elements for the period in which th --- +## Dialog modals + +Dialog modals are only used for having a conversation and confirmation with the user. The user is not able to access the features on the main page until closing the modal. + +### Usage + +* When the action is irreversible, dialog modals provide the details and confirm with the user before they take an advanced action. +* When the action will affect privacy or authorization, dialog modals provide advanced information and confirm with the user. + +### Style + +* Dialog modals contain the header, body, and actions. + * **Header(1):** The header title is a question instead of a descriptive phrase. + * **Body(2):** The content in body should never be ambiguous and unclear. It provides specific information. + * **Actions(3):** Contains a affirmative action, a dismissive action, and an extra action. The order of actions from left to right: Dismissive action → Extra action → Affirmative action +* Confirmations regarding labels should keep labeling styling. +* References to commits, branches, and tags should be **monospaced**. + + + +### Placement + +* Dialog modals should always be the center of the screen horizontally and be positioned **72px** from the top. + +| Dialog with 2 actions | Dialog with 3 actions | Special confirmation | +| --------------------- | --------------------- | -------------------- | +|  |  |  | + +> TODO: Special case for dialog modal. + +--- + ## Panels > TODO: Catalog how we are currently using panels and rationalize how they relate to alerts diff --git a/doc/development/ux_guide/img/modals-general-confimation-dialog.png b/doc/development/ux_guide/img/modals-general-confimation-dialog.png Binary files differnew file mode 100644 index 00000000000..00a17374a0b --- /dev/null +++ b/doc/development/ux_guide/img/modals-general-confimation-dialog.png diff --git a/doc/development/ux_guide/img/modals-layout-for-modals.png b/doc/development/ux_guide/img/modals-layout-for-modals.png Binary files differnew file mode 100644 index 00000000000..6c7bc09e750 --- /dev/null +++ b/doc/development/ux_guide/img/modals-layout-for-modals.png diff --git a/doc/development/ux_guide/img/modals-special-confimation-dialog.png b/doc/development/ux_guide/img/modals-special-confimation-dialog.png Binary files differnew file mode 100644 index 00000000000..bf1e56326c5 --- /dev/null +++ b/doc/development/ux_guide/img/modals-special-confimation-dialog.png diff --git a/doc/development/ux_guide/img/modals-three-buttons.png b/doc/development/ux_guide/img/modals-three-buttons.png Binary files differnew file mode 100644 index 00000000000..519439e64e4 --- /dev/null +++ b/doc/development/ux_guide/img/modals-three-buttons.png diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 2b305cb5c99..97e9b36d1a6 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -3,49 +3,47 @@ ## Notes - The owner of the project will be the first admin -- The groups will be created as needed +- The groups will be created as needed, including subgroups - The owner of the group will be the first admin - Existing projects will be skipped +- The existing Git repos will be moved from disk (removed from the original path) ## How to use -### Create a new folder inside the git repositories path. This will be the name of the new group. +### Create a new folder to import your Git repositories from. -- For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed -it in the `/etc/gitlab/gitlab.rb` file. -- For installations from source, it is usually located at: `/home/git/repositories` or you can see where -your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries -(you'll usually use the `default` storage path to start). - -New folder needs to have git user ownership and read/write/execute access for git user and its group: +The new folder needs to have git user ownership and read/write/execute access for git user and its group: ``` -sudo -u git mkdir /var/opt/gitlab/git-data/repositories/new_group +sudo -u git mkdir /var/opt/gitlab/git-data/repository-import-<date>/new_group ``` -If you are using an installation from source, replace `/var/opt/gitlab/git-data` -with `/home/git`. - ### Copy your bare repositories inside this newly created folder: +- Any .git repositories found on any of the subfolders will be imported as projects +- Groups will be created as needed, these could be nested folders. Example: + +If we copy the repos to `/var/opt/gitlab/git-data/repository-import-<date>`, and repo A needs to be under the groups G1 and G2, it will +have to be created under those folders: `/var/opt/gitlab/git-data/repository-import-<date>/G1/G2/A.git`. + + ``` -sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repositories/new_group/ +sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repository-import-<date>/new_group/ # Do this once when you are done copying git repositories -sudo chown -R git:git /var/opt/gitlab/git-data/repositories/new_group/ +sudo chown -R git:git /var/opt/gitlab/git-data/repository-import-<date> ``` `foo.git` needs to be owned by the git user and git users group. -If you are using an installation from source, replace `/var/opt/gitlab/git-data` -with `/home/git`. +If you are using an installation from source, replace `/var/opt/gitlab/` with `/home/git`. ### Run the command below depending on your type of installation: #### Omnibus Installation ``` -$ sudo gitlab-rake gitlab:import:repos +$ sudo gitlab-rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] ``` #### Installation from source @@ -54,16 +52,21 @@ Before running this command you need to change the directory to where your GitLa ``` $ cd /home/git/gitlab -$ sudo -u git -H bundle exec rake gitlab:import:repos RAILS_ENV=production +$ sudo -u git -H bundle exec rake gitlab:import:repos['/var/opt/gitlab/git-data/repository-import-<date>'] RAILS_ENV=production ``` #### Example output ``` -Processing abcd.git +Processing /var/opt/gitlab/git-data/repository-import-1/a/b/c/blah.git + * Using namespace: a/b/c + * Created blah (a/b/c/blah) + * Skipping repo /var/opt/gitlab/git-data/repository-import-1/a/b/c/blah.wiki.git +Processing /var/opt/gitlab/git-data/repository-import-1/abcd.git * Created abcd (abcd.git) -Processing group/xyz.git - * Created Group group (2) +Processing /var/opt/gitlab/git-data/repository-import-1/group/xyz.git + * Using namespace: group (2) * Created xyz (group/xyz.git) + * Skipping repo /var/opt/gitlab/git-data/repository-import-1/@shared/a/b/abcd.git [...] ``` diff --git a/doc/user/project/clusters/img/cluster-applications.png b/doc/user/project/clusters/img/cluster-applications.png Binary files differnew file mode 100644 index 00000000000..7c82d19297e --- /dev/null +++ b/doc/user/project/clusters/img/cluster-applications.png diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 7d9e771f570..27b4b49c207 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -88,3 +88,12 @@ To remove the Cluster integration from your project, simply click on the and [add a cluster](#adding-a-cluster) again. [permissions]: ../../permissions.md + +## Installing applications + +GitLab provides a one-click install for +[Helm Tiller](https://docs.helm.sh/) and +[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) +which will be added directly to your configured cluster. + + diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 2c4dfcff4a6..394aa9209e4 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -27,7 +27,8 @@ to enable it. 1. First, ask your system administrator to enable GitLab Container Registry following the [administration documentation](../../administration/container_registry.md). If you are using GitLab.com, this is enabled by default so you can start using - the Registry immediately. + the Registry immediately. Currently there is a soft (10GB) size restriction for + registry on GitLab.com, as part of the [repository size limit](repository/index.html#repository-size). 1. Go to your [project's General settings](settings/index.md#sharing-and-permissions) and enable the **Container Registry** feature on your project. For new projects this might be enabled by default. For existing projects diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 6423beefc77..72def9d1d1d 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -24,6 +24,8 @@ constrains of a Sidekiq worker. - the milestones (GitLab 8.7+) - the labels (GitLab 8.7+) - the release note descriptions (GitLab 8.12+) + - the pull request review comments (GitLab 10.2+) + - the regular issue and pull request comments - References to pull requests and issues are preserved (GitLab 8.7+) - Repository public access is retained. If a repository is private in GitHub it will be created as private in GitLab as well. @@ -43,10 +45,13 @@ the case the namespace is taken, the repository will be imported under the user' namespace that started the import process. The importer will also import branches on forks of projects related to open pull -requests. These branches will be imported with a naming scheume similar to +requests. These branches will be imported with a naming scheme similar to GH-SHA-Username/Pull-Request-number/fork-name/branch. This may lead to a discrepency in branches compared to the GitHub Repository. +For a more technical description and an overview of the architecture you can +refer to [Working with the GitHub importer][gh-import-dev-docs]. + ## Importing your GitHub repositories The importer page is visible when you create a new project. @@ -121,7 +126,29 @@ If you want, you can import all your GitHub projects in one go by hitting You can also choose a different name for the project and a different namespace, if you have the privileges to do so. +## Making the import process go faster + +For large projects it may take a while to import all data. To reduce the time +necessary you can increase the number of Sidekiq workers that process the +following queues: + +* `github_importer` +* `github_importer_advance_stage` + +For an optimal experience we recommend having at least 4 Sidekiq processes (each +running a number of threads equal to the number of CPU cores) that _only_ +process these queues. We also recommend that these processes run on separate +servers. For 4 servers with 8 cores this means you can import up to 32 objects +(e.g. issues) in parallel. + +Reducing the time spent in cloning a repository can be done by increasing +network throughput, CPU capacity, and disk performance (e.g. by using high +performance SSDs) of the disks that store the Git repositories (for your GitLab +instance). Increasing the number of Sidekiq workers will _not_ reduce the time +spent cloning repositories. + [gh-import]: ../../../integration/github.md "GitHub integration" [gh-rake]: ../../../administration/raketasks/github_import.md "GitHub rake task" [gh-integration]: #authorize-access-to-your-repositories-using-the-github-integration [gh-token]: #authorize-access-to-your-repositories-using-a-personal-access-token +[gh-import-dev-docs]: ../../../development/github_importer.md "Working with the GitHub importer" diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 96a5a23ee13..8c2690ec3b2 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -166,12 +166,26 @@ board itself.  -## Re-ordering an issue in a list - -> Introduced in GitLab 9.0. - -Issues can be re-ordered inside of lists. This is as simple as dragging and dropping -an issue into the order you want. +## Issue ordering in a list + +When visiting a board, issues appear ordered in any list. You are able to change +that order simply by dragging and dropping the issues. The changed order will be saved +to the system so that anybody who visits the same board later will see the reordering, +with some exceptions. + +The first time a given issue appears in any board (i.e. the first time a user +loads a board containing that issue), it will be ordered with +respect to other issues in that list according to [Priority order][label-priority]. +At that point, that issue will be assigned a relative order value by the system +representing its relative order with respect to the other issues in the list. Any time +you drag-and-drop reorder that issue, its relative order value will change accordingly. +Also, any time that issue appears in any board when it is loaded by a user, +the updated relative order value will be used for the ordering. (It's only the first +time an issue appears that it takes from the Priority order mentioned above.) This means that +if issue `A` is drag-and-drop reordered to be above issue `B` by any user in +a given board inside your GitLab instance, any time those two issues are subsequently +loaded in any board in the same instance (could be a different project board or a different group board, for example), +that ordering will be maintained. ## Filtering issues diff --git a/lib/feature.rb b/lib/feature.rb index 4bd29aed687..ac3bc65c0d5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -5,6 +5,10 @@ class Feature class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature # Using `self.table_name` won't work. ActiveRecord bug? superclass.table_name = 'features' + + def self.feature_names + pluck(:key) + end end class FlipperGate < Flipper::Adapters::ActiveRecord::Gate @@ -22,11 +26,19 @@ class Feature flipper.feature(key) end + def persisted_names + if RequestStore.active? + RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names + else + FlipperFeature.feature_names + end + end + def persisted?(feature) # Flipper creates on-memory features when asked for a not-yet-created one. # If we want to check if a feature has been actually set, we look for it # on the persisted features list. - all.map(&:name).include?(feature.name) + persisted_names.include?(feature.name) end def enabled?(key, thing = nil) diff --git a/lib/github/client.rb b/lib/github/client.rb deleted file mode 100644 index 29bd9c1f39e..00000000000 --- a/lib/github/client.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Github - class Client - TIMEOUT = 60 - DEFAULT_PER_PAGE = 100 - - attr_reader :connection, :rate_limit - - def initialize(options) - @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday| - faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT) - faraday.options.timeout = options.fetch(:timeout, TIMEOUT) - faraday.authorization 'token', options.fetch(:token) - faraday.adapter :net_http - faraday.ssl.verify = verify_ssl - end - - @rate_limit = RateLimit.new(connection) - end - - def get(url, query = {}) - exceed, reset_in = rate_limit.get - sleep reset_in if exceed - - Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) - end - - private - - def root_endpoint - custom_endpoint || github_endpoint - end - - def custom_endpoint - github_omniauth_provider.dig('args', 'client_options', 'site') - end - - def verify_ssl - # If there is no config, we're connecting to github.com - # and we should verify ssl. - github_omniauth_provider.fetch('verify_ssl', true) - end - - def github_endpoint - OmniAuth::Strategies::GitHub.default_options[:client_options][:site] - end - - def github_omniauth_provider - @github_omniauth_provider ||= - Gitlab.config.omniauth.providers - .find { |provider| provider.name == 'github' } - .to_h - end - end -end diff --git a/lib/github/collection.rb b/lib/github/collection.rb deleted file mode 100644 index 014b2038c4b..00000000000 --- a/lib/github/collection.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Github - class Collection - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch(url, query = {}) - return [] if url.blank? - - Enumerator.new do |yielder| - loop do - response = client.get(url, query) - response.body.each { |item| yielder << item } - - raise StopIteration unless response.rels.key?(:next) - url = response.rels[:next] - end - end.lazy - end - - private - - def client - @client ||= Github::Client.new(options) - end - end -end diff --git a/lib/github/error.rb b/lib/github/error.rb deleted file mode 100644 index 66d7afaa787..00000000000 --- a/lib/github/error.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Github - RepositoryFetchError = Class.new(StandardError) -end diff --git a/lib/github/import.rb b/lib/github/import.rb deleted file mode 100644 index fef63dd7168..00000000000 --- a/lib/github/import.rb +++ /dev/null @@ -1,377 +0,0 @@ -require_relative 'error' -require_relative 'import/issue' -require_relative 'import/legacy_diff_note' -require_relative 'import/merge_request' -require_relative 'import/note' - -module Github - class Import - include Gitlab::ShellAdapter - - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose, :last_fetched_at - - def initialize(project, options = {}) - @project = project - @repository = project.repository - @repo = project.import_source - @repo_url = project.import_url - @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user)) - @verbose = options.fetch(:verbose, false) - @cached = Hash.new { |hash, key| hash[key] = Hash.new } - @errors = [] - @last_fetched_at = nil - end - - # rubocop: disable Rails/Output - def execute - puts 'Fetching repository...'.color(:aqua) if verbose - setup_and_fetch_repository - puts 'Fetching labels...'.color(:aqua) if verbose - fetch_labels - puts 'Fetching milestones...'.color(:aqua) if verbose - fetch_milestones - puts 'Fetching pull requests...'.color(:aqua) if verbose - fetch_pull_requests - puts 'Fetching issues...'.color(:aqua) if verbose - fetch_issues - puts 'Fetching releases...'.color(:aqua) if verbose - fetch_releases - puts 'Cloning wiki repository...'.color(:aqua) if verbose - fetch_wiki_repository - puts 'Expiring repository cache...'.color(:aqua) if verbose - expire_repository_cache - - errors.empty? - rescue Github::RepositoryFetchError - expire_repository_cache - false - ensure - keep_track_of_errors - end - - private - - def setup_and_fetch_repository - begin - project.ensure_repository - project.repository.add_remote('github', repo_url) - project.repository.set_import_remote_as_mirror('github') - project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') - fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, - Gitlab::Git::RepositoryMirroring::RemoteError, - Gitlab::Shell::Error => e - error(:project, repo_url, e.message) - raise Github::RepositoryFetchError - end - end - - def fetch_remote(forced: false) - @last_fetched_at = Time.now - project.repository.fetch_remote('github', forced: forced) - end - - def fetch_wiki_repository - return if project.wiki.repository_exists? - - wiki_path = project.wiki.disk_path - gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) - rescue Gitlab::Shell::Error => e - # GitHub error message when the wiki repo has not been created, - # this means that repo has wiki enabled, but have no pages. So, - # we can skip the import. - if e.message !~ /repository not exported/ - error(:wiki, wiki_url, e.message) - end - end - - def fetch_labels - url = "/repos/#{repo}/labels" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - begin - representation = Github::Representation::Label.new(raw) - - label = project.labels.find_or_create_by!(title: representation.title) do |label| - label.color = representation.color - end - - cached[:label_ids][representation.title] = label.id - rescue => e - error(:label, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_milestones - url = "/repos/#{repo}/milestones" - - while url - response = Github::Client.new(options).get(url, state: :all) - - response.body.each do |raw| - begin - milestone = Github::Representation::Milestone.new(raw) - next if project.milestones.where(iid: milestone.iid).exists? - - project.milestones.create!( - iid: milestone.iid, - title: milestone.title, - description: milestone.description, - due_date: milestone.due_date, - state: milestone.state, - created_at: milestone.created_at, - updated_at: milestone.updated_at - ) - rescue => e - error(:milestone, milestone.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_pull_requests - url = "/repos/#{repo}/pulls" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) - merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) - next unless merge_request.new_record? && pull_request.valid? - - begin - # If the PR has been created/updated after we last fetched the - # remote, we fetch again to get the up-to-date refs. - fetch_remote if pull_request.updated_at > last_fetched_at - - author_id = user_id(pull_request.author, project.creator_id) - description = format_description(pull_request.description, pull_request.author) - - merge_request.attributes = { - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: pull_request.source_project, - source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, - target_project: pull_request.target_project, - target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, - state: pull_request.state, - milestone_id: milestone_id(pull_request.milestone), - author_id: author_id, - assignee_id: user_id(pull_request.assignee), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - merge_request.save!(validate: false) - merge_request.merge_request_diffs.create - - review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" - fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - rescue => e - error(:pull_request, pull_request.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_issues - url = "/repos/#{repo}/issues" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each { |raw| populate_issue(raw) } - - url = response.rels[:next] - end - end - - def populate_issue(raw) - representation = Github::Representation::Issue.new(raw, options) - - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - return unless representation.labels? || representation.comments? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - - if representation.labels? - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - end - - fetch_comments_conditionally(merge_request, representation) - else - return if Issue.exists?(iid: representation.iid, project_id: project.id) - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - issue.update( - label_ids: label_ids(representation.labels), - assignee_ids: assignee_ids(representation.assignees)) - - fetch_comments_conditionally(issue, representation) - end - rescue => e - error(:issue, representation.url, e.message) - end - end - - def fetch_comments_conditionally(issuable, representation) - if representation.comments? - comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" - fetch_comments(issuable, :comment, comments_url) - end - end - - def fetch_comments(noteable, type, url, klass = Note) - while url - comments = Github::Client.new(options).get(url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - representation = Github::Representation::Comment.new(raw, options) - author_id = user_id(representation.author, project.creator_id) - - note = klass.new - note.project_id = project.id - note.noteable = noteable - note.note = format_description(representation.note, representation.author) - note.commit_id = representation.commit_id - note.line_code = representation.line_code - note.author_id = author_id - note.created_at = representation.created_at - note.updated_at = representation.updated_at - note.save!(validate: false) - rescue => e - error(type, representation.url, e.message) - end - end - end - - url = comments.rels[:next] - end - end - - def fetch_releases - url = "/repos/#{repo}/releases" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - representation = Github::Representation::Release.new(raw) - next unless representation.valid? - - release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) - next unless release.new_record? - - begin - release.description = representation.description - release.created_at = representation.created_at - release.updated_at = representation.updated_at - release.save!(validate: false) - rescue => e - error(:release, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def label_ids(labels) - labels.map { |label| cached[:label_ids][label.title] }.compact - end - - def assignee_ids(assignees) - assignees.map { |assignee| user_id(assignee) }.compact - end - - def milestone_id(milestone) - return unless milestone.present? - - project.milestones.select(:id).find_by(iid: milestone.iid)&.id - end - - def user_id(user, fallback_id = nil) - return unless user.present? - return cached[:user_ids][user.id] if cached[:user_ids][user.id].present? - - gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) - - cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? - cached[:user_ids][user.id] = gitlab_user_id || fallback_id - end - - def user_id_by_email(email) - return nil unless email - - ::User.find_by_any_email(email)&.id - end - - def user_id_by_external_uid(id) - return nil unless id - - ::User.select(:id) - .joins(:identities) - .merge(::Identity.where(provider: :github, extern_uid: id)) - .first&.id - end - - def format_description(body, author) - return body if cached[:gitlab_user_ids][author.id] - - "*Created by: #{author.username}*\n\n#{body}" - end - - def expire_repository_cache - repository.expire_content_cache if project.repository_exists? - end - - def keep_track_of_errors - return unless errors.any? - - project.update_column(:import_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def error(type, url, message) - errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } - end - end -end diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb deleted file mode 100644 index 171f0872666..00000000000 --- a/lib/github/import/issue.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb deleted file mode 100644 index 18adff560b6..00000000000 --- a/lib/github/import/legacy_diff_note.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Github - class Import - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb deleted file mode 100644 index c258e5d5e0e..00000000000 --- a/lib/github/import/merge_request.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb deleted file mode 100644 index 8cf4f30e6b7..00000000000 --- a/lib/github/import/note.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Note < ::Note - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb deleted file mode 100644 index 884693d093c..00000000000 --- a/lib/github/rate_limit.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - class RateLimit - SAFE_REMAINING_REQUESTS = 100 - SAFE_RESET_TIME = 500 - RATE_LIMIT_URL = '/rate_limit'.freeze - - attr_reader :connection - - def initialize(connection) - @connection = connection - end - - def get - response = connection.get(RATE_LIMIT_URL) - - # GitHub Rate Limit API returns 404 when the rate limit is disabled - return false unless response.status != 404 - - body = Oj.load(response.body, class_cache: false, mode: :compat) - remaining = body.dig('rate', 'remaining').to_i - reset_in = body.dig('rate', 'reset').to_i - exceed = remaining <= SAFE_REMAINING_REQUESTS - - [exceed, reset_in] - end - end -end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb deleted file mode 100644 index c1c9448f305..00000000000 --- a/lib/github/repositories.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Github - class Repositories - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch - Collection.new(options).fetch(repos_url) - end - - private - - def repos_url - '/user/repos' - end - end -end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb deleted file mode 100644 index f26bdbdd546..00000000000 --- a/lib/github/representation/base.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Github - module Representation - class Base - def initialize(raw, options = {}) - @raw = raw - @options = options - end - - def id - raw['id'] - end - - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - - private - - attr_reader :raw, :options - end - end -end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb deleted file mode 100644 index 0087a3d3c4f..00000000000 --- a/lib/github/representation/branch.rb +++ /dev/null @@ -1,55 +0,0 @@ -module Github - module Representation - class Branch < Representation::Base - attr_reader :repository - - def user - raw.dig('user', 'login') || 'unknown' - end - - def repo? - raw['repo'].present? - end - - def repo - return unless repo? - - @repo ||= Github::Representation::Repo.new(raw['repo']) - end - - def ref - raw['ref'] - end - - def sha - raw['sha'] - end - - def short_sha - Commit.truncate_sha(sha) - end - - def valid? - sha.present? && ref.present? - end - - def restore!(name) - repository.create_branch(name, sha) - rescue Gitlab::Git::Repository::InvalidRef => e - Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}") - end - - def remove!(name) - repository.delete_branch(name) - rescue Gitlab::Git::Repository::DeleteBranchError => e - Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") - end - - private - - def repository - @repository ||= options.fetch(:repository) - end - end - end -end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb deleted file mode 100644 index 83bf0b5310d..00000000000 --- a/lib/github/representation/comment.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Github - module Representation - class Comment < Representation::Base - def note - raw['body'] || '' - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def commit_id - raw['commit_id'] - end - - def line_code - return unless on_diff? - - parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) - generate_line_code(parsed_lines.to_a.last) - end - - private - - def generate_line_code(line) - Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) - end - - def on_diff? - diff_hunk.present? - end - - def diff_hunk - raw['diff_hunk'] - end - - def file_path - raw['path'] - end - end - end -end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb deleted file mode 100644 index 768ba3b993c..00000000000 --- a/lib/github/representation/issuable.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Github - module Representation - class Issuable < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def labels? - raw['labels'].any? - end - - def labels - @labels ||= Array(raw['labels']).map do |label| - Github::Representation::Label.new(label, options) - end - end - end - end -end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb deleted file mode 100644 index 4f1a02cb90f..00000000000 --- a/lib/github/representation/issue.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - module Representation - class Issue < Representation::Issuable - def state - raw['state'] == 'closed' ? 'closed' : 'opened' - end - - def comments? - raw['comments'] > 0 - end - - def pull_request? - raw['pull_request'].present? - end - - def assigned? - raw['assignees'].present? - end - - def assignees - @assignees ||= Array(raw['assignees']).map do |user| - Github::Representation::User.new(user, options) - end - end - end - end -end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb deleted file mode 100644 index 60aa51f9569..00000000000 --- a/lib/github/representation/label.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - module Representation - class Label < Representation::Base - def color - "##{raw['color']}" - end - - def title - raw['name'] - end - end - end -end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb deleted file mode 100644 index 917e6394ad4..00000000000 --- a/lib/github/representation/milestone.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - module Representation - class Milestone < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['description'] - end - - def due_date - raw['due_on'] - end - - def state - raw['state'] == 'closed' ? 'closed' : 'active' - end - end - end -end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb deleted file mode 100644 index 0171179bb0f..00000000000 --- a/lib/github/representation/pull_request.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Github - module Representation - class PullRequest < Representation::Issuable - delegate :sha, to: :source_branch, prefix: true - delegate :sha, to: :target_branch, prefix: true - - def source_project - project - end - - def source_branch_name - # Mimic the "user:branch" displayed in the MR widget, - # i.e. "Request to merge rymai:add-external-mounts into master" - cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref - end - - def target_project - project - end - - def target_branch_name - target_branch.ref - end - - def state - return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? - return 'closed' if raw['state'] == 'closed' - - 'opened' - end - - def opened? - state == 'opened' - end - - def valid? - source_branch.valid? && target_branch.valid? - end - - def assigned? - raw['assignee'].present? - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) - end - - private - - def project - @project ||= options.fetch(:project) - end - - def source_branch - @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) - end - - def target_branch - @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) - end - - def cross_project? - return true unless source_branch.repo? - - source_branch.repo.id != target_branch.repo.id - end - end - end -end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb deleted file mode 100644 index e7e4b428c1a..00000000000 --- a/lib/github/representation/release.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Github - module Representation - class Release < Representation::Base - def description - raw['body'] - end - - def tag - raw['tag_name'] - end - - def valid? - !raw['draft'] - end - end - end -end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb deleted file mode 100644 index 6938aa7db05..00000000000 --- a/lib/github/representation/repo.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Github - module Representation - class Repo < Representation::Base - end - end -end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb deleted file mode 100644 index 18591380e25..00000000000 --- a/lib/github/representation/user.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Github - module Representation - class User < Representation::Base - def email - return @email if defined?(@email) - - @email = Github::User.new(username, options).get.fetch('email', nil) - end - - def username - raw['login'] - end - end - end -end diff --git a/lib/github/response.rb b/lib/github/response.rb deleted file mode 100644 index 761c524b553..00000000000 --- a/lib/github/response.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - class Response - attr_reader :raw, :headers, :status - - def initialize(response) - @raw = response - @headers = response.headers - @status = response.status - end - - def body - Oj.load(raw.body, class_cache: false, mode: :compat) - end - - def rels - links = headers['Link'].to_s.split(', ').map do |link| - href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures - - [name.to_sym, href] - end - - Hash[*links.flatten] - end - end -end diff --git a/lib/github/user.rb b/lib/github/user.rb deleted file mode 100644 index f88a29e590b..00000000000 --- a/lib/github/user.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Github - class User - attr_reader :username, :options - - def initialize(username, options) - @username = username - @options = options - end - - def get - client.get(user_url).body - end - - private - - def client - @client ||= Github::Client.new(options) - end - - def user_url - "/users/#{username}" - end - end -end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb new file mode 100644 index 00000000000..196de667805 --- /dev/null +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -0,0 +1,101 @@ +module Gitlab + module BareRepositoryImport + class Importer + NoAdminError = Class.new(StandardError) + + def self.execute(import_path) + import_path << '/' unless import_path.ends_with?('/') + repos_to_import = Dir.glob(import_path + '**/*.git') + + unless user = User.admins.order_id_asc.first + raise NoAdminError.new('No admin user found to import repositories') + end + + repos_to_import.each do |repo_path| + bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) + + if bare_repo.hashed? || bare_repo.wiki? + log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) + + next + end + + log "Processing #{repo_path}".color(:yellow) + + new(user, bare_repo).create_project_if_needed + end + end + + attr_reader :user, :project_name, :bare_repo + + delegate :log, to: :class + delegate :project_name, :project_full_path, :group_path, :repo_path, :wiki_path, to: :bare_repo + + def initialize(user, bare_repo) + @user = user + @bare_repo = bare_repo + end + + def create_project_if_needed + if project = Project.find_by_full_path(project_full_path) + log " * #{project.name} (#{project_full_path}) exists" + + return project + end + + create_project + end + + private + + def create_project + group = find_or_create_groups + + project = Projects::CreateService.new(user, + name: project_name, + path: project_name, + skip_disk_validation: true, + namespace_id: group&.id).execute + + if project.persisted? && mv_repo(project) + log " * Created #{project.name} (#{project_full_path})".color(:green) + + ProjectCacheWorker.perform_async(project.id) + else + log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) + log " Errors: #{project.errors.messages}".color(:red) if project.errors.any? + end + + project + end + + def mv_repo(project) + FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git')) + + if bare_repo.wiki_exists? + FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + end + + true + rescue => e + log " * Failed to move repo: #{e.message}".color(:red) + + false + end + + def find_or_create_groups + return nil unless group_path.present? + + log " * Using namespace: #{group_path}" + + Groups::NestedCreateService.new(user, group_path: group_path).execute + end + + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb new file mode 100644 index 00000000000..8574ac6eb30 --- /dev/null +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -0,0 +1,42 @@ +module Gitlab + module BareRepositoryImport + class Repository + attr_reader :group_path, :project_name, :repo_path + + def initialize(root_path, repo_path) + @root_path = root_path + @repo_path = repo_path + + # Split path into 'all/the/namespaces' and 'project_name' + @group_path, _, @project_name = repo_relative_path.rpartition('/') + end + + def wiki_exists? + File.exist?(wiki_path) + end + + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def wiki_path + @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') + end + + def hashed? + @hashed ||= group_path.start_with?('@hashed') + end + + def project_full_path + @project_full_path ||= "#{group_path}/#{project_name}" + end + + private + + def repo_relative_path + # Remove root path and `.git` at the end + repo_path[@root_path.size...-4] + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb deleted file mode 100644 index 1d98d187805..00000000000 --- a/lib/gitlab/bare_repository_importer.rb +++ /dev/null @@ -1,97 +0,0 @@ -module Gitlab - class BareRepositoryImporter - NoAdminError = Class.new(StandardError) - - def self.execute - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - git_base_path = repository_storage['path'] - repos_to_import = Dir.glob(git_base_path + '/**/*.git') - - repos_to_import.each do |repo_path| - if repo_path.end_with?('.wiki.git') - log " * Skipping wiki repo" - next - end - - log "Processing #{repo_path}".color(:yellow) - - repo_relative_path = repo_path[repository_storage['path'].length..-1] - .sub(/^\//, '') # Remove leading `/` - .sub(/\.git$/, '') # Remove `.git` at the end - new(storage_name, repo_relative_path).create_project_if_needed - end - end - end - - attr_reader :storage_name, :full_path, :group_path, :project_path, :user - delegate :log, to: :class - - def initialize(storage_name, repo_path) - @storage_name = storage_name - @full_path = repo_path - - unless @user = User.admins.order_id_asc.first - raise NoAdminError.new('No admin user found to import repositories') - end - - @group_path, @project_path = File.split(repo_path) - @group_path = nil if @group_path == '.' - end - - def create_project_if_needed - if project = Project.find_by_full_path(full_path) - log " * #{project.name} (#{full_path}) exists" - return project - end - - create_project - end - - private - - def create_project - group = find_or_create_group - - project_params = { - name: project_path, - path: project_path, - repository_storage: storage_name, - namespace_id: group&.id, - skip_disk_validation: true - } - - project = Projects::CreateService.new(user, project_params).execute - - if project.persisted? - log " * Created #{project.name} (#{full_path})".color(:green) - ProjectCacheWorker.perform_async(project.id) - else - log " * Failed trying to create #{project.name} (#{full_path})".color(:red) - log " Errors: #{project.errors.messages}".color(:red) - end - - project - end - - def find_or_create_group - return nil unless group_path - - if namespace = Namespace.find_by_full_path(group_path) - log " * Namespace #{group_path} exists.".color(:green) - return namespace - end - - log " * Creating Group: #{group_path}" - Groups::NestedCreateService.new(user, group_path: group_path).execute - end - - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - # - # rubocop:disable Rails/Output - def self.log(message) - puts message - end - # rubocop:enable Rails/Output - end -end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index d71e63e73eb..dc90f398c7e 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -8,7 +8,7 @@ module Gitlab end def icon - 'warning' + 'status_warning' end def group diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 43a00d6cedb..cd7b4c043da 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -108,20 +108,41 @@ module Gitlab end end - def self.bulk_insert(table, rows) + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows, this only works on PostgreSQL. + def self.bulk_insert(table, rows, return_ids: false) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } + return_ids = false if mysql? tuples = rows.map do |row| row.values_at(*keys).map { |value| connection.quote(value) } end - connection.execute <<-EOF + sql = <<-EOF INSERT INTO #{table} (#{columns.join(', ')}) VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF + + if return_ids + sql << 'RETURNING id' + end + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end end def self.sanitize_timestamp(timestamp) diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index aee3981e79a..9f76967fc77 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,28 +6,36 @@ module Gitlab if Database.postgresql? 'information_schema.role_table_grants' else - 'mysql.user' + 'information_schema.schema_privileges' end - def self.scope_to_current_user - if Database.postgresql? - where('grantee = user') - else - where("CONCAT(User, '@', Host) = current_user()") - end - end - # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) priv = if Database.postgresql? where(privilege_type: 'TRIGGER', table_name: table) + .where('grantee = user') else - where(Trigger_priv: 'Y') + queries = [ + Grant.select(1) + .from('information_schema.user_privileges') + .where("PRIVILEGE_TYPE = 'SUPER'") + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), + + Grant.select(1) + .from('information_schema.schema_privileges') + .where("PRIVILEGE_TYPE = 'TRIGGER'") + .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") + ] + + union = SQL::Union.new(queries).to_sql + + Grant.from("(#{union}) privs") end - priv.scope_to_current_user.any? + priv.any? end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index df4ad586e12..d236e1b03e6 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -920,6 +920,11 @@ module Gitlab false end + # Returns true if a remote exists. + def remote_exists?(name) + rugged.remotes[name].present? + end + # Update the specified remote using the values in the +options+ hash # # Example diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb new file mode 100644 index 00000000000..d2ae4c1255e --- /dev/null +++ b/lib/gitlab/github_import.rb @@ -0,0 +1,34 @@ +module Gitlab + module GithubImport + def self.new_client_for(project, token: nil, parallel: true) + token_to_use = token || project.import_data&.credentials&.fetch(:user) + + Client.new(token_to_use, parallel: parallel) + end + + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + def self.insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + + # Returns the ID of the ghost user. + def self.ghost_user_id + key = 'github-import/ghost-user-id' + + Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id) + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb new file mode 100644 index 00000000000..147597289cf --- /dev/null +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module BulkImporting + # Builds and returns an Array of objects to bulk insert into the + # database. + # + # enum - An Enumerable that returns the objects to turn into database + # rows. + def build_database_rows(enum) + enum.each_with_object([]) do |(object, _), rows| + rows << build(object) unless already_imported?(object) + end + end + + # Bulk inserts the given rows into the database. + def bulk_insert(model, rows, batch_size: 100) + rows.each_slice(batch_size) do |slice| + Gitlab::Database.bulk_insert(model.table_name, slice) + end + end + end + end +end diff --git a/lib/gitlab/github_import/caching.rb b/lib/gitlab/github_import/caching.rb new file mode 100644 index 00000000000..b08f133794f --- /dev/null +++ b/lib/gitlab/github_import/caching.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Caching + # The default timeout of the cache keys. + TIMEOUT = 24.hours.to_i + + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze + local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] + local existing = tonumber(redis.call("get", key)) + + if existing == nil or value > existing then + redis.call("set", key, value) + redis.call("expire", key, ttl) + return true + else + return false + end + EOF + + # Reads a cache key. + # + # If the key exists and has a non-empty value its TTL is refreshed + # automatically. + # + # raw_key - The cache key to read. + # timeout - The new timeout of the key if the key is to be refreshed. + def self.read(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + value = Redis::Cache.with { |redis| redis.get(key) } + + if value.present? + # We refresh the expiration time so frequently used keys stick + # around, removing the need for querying the database as much as + # possible. + # + # A key may be empty when we looked up a GitHub user (for example) but + # did not find a matching GitLab user. In that case we _don't_ want to + # refresh the TTL so we automatically pick up the right data when said + # user were to register themselves on the GitLab instance. + Redis::Cache.with { |redis| redis.expire(key, timeout) } + end + + value + end + + # Reads an integer from the cache, or returns nil if no value was found. + # + # See Caching.read for more information. + def self.read_integer(raw_key, timeout: TIMEOUT) + value = read(raw_key, timeout: timeout) + + value.to_i if value.present? + end + + # Sets a cache key to the given value. + # + # key - The cache key to write. + # value - The value to set. + # timeout - The time after which the cache key should expire. + def self.write(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.set(key, value, ex: timeout) + end + + value + end + + # Adds a value to a set. + # + # raw_key - The key of the set to add the value to. + # value - The value to add to the set. + # timeout - The new timeout of the key. + def self.set_add(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.sadd(key, value) + m.expire(key, timeout) + end + end + end + + # Returns true if the given value is present in the set. + # + # raw_key - The key of the set to check. + # value - The value to check for. + def self.set_includes?(raw_key, value) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.sismember(key, value) + end + end + + # Sets multiple keys to a given value. + # + # mapping - A Hash mapping the cache keys to their values. + # timeout - The time after which the cache key should expire. + def self.write_multiple(mapping, timeout: TIMEOUT) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + multi.set(cache_key_for(raw_key), value, ex: timeout) + end + end + end + end + + # Sets the expiration time of a key. + # + # raw_key - The key for which to change the timeout. + # timeout - The new timeout. + def self.expire(raw_key, timeout) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.expire(key, timeout) + end + end + + # Sets a key to the given integer but only if the existing value is + # smaller than the given value. + # + # This method uses a Lua script to ensure the read and write are atomic. + # + # raw_key - The key to set. + # value - The new value for the key. + # timeout - The key timeout in seconds. + # + # Returns true when the key was overwritten, false otherwise. + def self.write_if_greater(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + val = Redis::Cache.with do |redis| + redis + .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) + end + + val ? true : false + end + + def self.cache_key_for(raw_key) + "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0550f9695bd..844530b1ea7 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,147 +1,184 @@ +# frozen_string_literal: true + module Gitlab module GithubImport + # HTTP client for interacting with the GitHub API. + # + # This class is basically a fancy wrapped around Octokit while adding some + # functionality to deal with rate limiting and parallel imports. Usage is + # mostly the same as Octokit, for example: + # + # client = GithubImport::Client.new('hunter2') + # + # client.labels.each do |label| + # puts label.name + # end class Client - GITHUB_SAFE_REMAINING_REQUESTS = 100 - GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :octokit - attr_reader :access_token, :host, :api_version + # A single page of data and the corresponding page number. + Page = Struct.new(:objects, :number) - def initialize(access_token, host: nil, api_version: 'v3') - @access_token = access_token - @host = host.to_s.sub(%r{/+\z}, '') - @api_version = api_version - @users = {} + # The minimum number of requests we want to keep available. + # + # We don't use a value of 0 as multiple threads may be using the same + # token in parallel. This could result in all of them hitting the GitHub + # rate limit at once. The threshold is put in place to not hit the limit + # in most cases. + RATE_LIMIT_THRESHOLD = 50 - if access_token - ::Octokit.auto_paginate = false - end + # token - The GitHub API token to use. + # + # per_page - The number of objects that should be displayed per page. + # + # parallel - When set to true hitting the rate limit will result in a + # dedicated error being raised. When set to `false` we will + # instead just `sleep()` until the rate limit is reset. Setting + # this value to `true` for parallel importing is crucial as + # otherwise hitting the rate limit will result in a thread + # being blocked in a `sleep()` call for up to an hour. + def initialize(token, per_page: 100, parallel: true) + @octokit = Octokit::Client.new(access_token: token, per_page: per_page) + @parallel = parallel end - def api - @api ||= ::Octokit::Client.new( - access_token: access_token, - api_endpoint: api_endpoint, - # If there is no config, we're connecting to github.com and we - # should verify ssl. - connection_options: { - ssl: { verify: config ? config['verify_ssl'] : true } - } - ) + def parallel? + @parallel end - def client - unless config - raise Projects::ImportService::Error, - 'OAuth configuration for GitHub missing.' - end - - @client ||= ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + # Returns the details of a GitHub user. + # + # username - The username of the user. + def user(username) + with_rate_limit { octokit.user(username) } end - def authorize_url(redirect_uri) - client.auth_code.authorize_url({ - redirect_uri: redirect_uri, - scope: "repo, user, user:email" - }) + # Returns the details of a GitHub repository. + # + # name - The path (in the form `owner/repository`) of the repository. + def repository(name) + with_rate_limit { octokit.repo(name) } end - def get_token(code) - client.auth_code.get_token(code).token + def labels(*args) + each_object(:labels, *args) end - def method_missing(method, *args, &block) - if api.respond_to?(method) - request(method, *args, &block) - else - super(method, *args, &block) - end + def milestones(*args) + each_object(:milestones, *args) end - def respond_to?(method) - api.respond_to?(method) || super + def releases(*args) + each_object(:releases, *args) end - def user(login) - return nil unless login.present? - return @users[login] if @users.key?(login) + # Fetches data from the GitHub API and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the Octokit method. + # + # rubocop: disable GitlabSecurity/PublicSend + def each_page(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? - @users[login] = api.user(login) - end + page = + if args.last.is_a?(Hash) && args.last[:page] + args.last[:page] + else + 1 + end - private + collection = with_rate_limit { octokit.public_send(method, *args) } + next_url = octokit.last_response.rels[:next] - def api_endpoint - if host.present? && api_version.present? - "#{host}/api/#{api_version}" - else - github_options[:site] + yield Page.new(collection, page) + + while next_url + response = with_rate_limit { next_url.get } + next_url = response.rels[:next] + + yield Page.new(response.data, page += 1) end end - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } - end + # Iterates over all of the objects for the given method (e.g. `:labels`). + # + # method - The method to send to Octokit for querying data. + # args - Any arguments to pass to the Octokit method. + def each_object(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? - def github_options - if config - config["args"]["client_options"].deep_symbolize_keys - else - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + each_page(method, *args) do |page| + page.objects.each do |object| + yield object + end end end - def rate_limit - api.rate_limit! - # GitHub Rate Limit API returns 404 when the rate limit is - # disabled. In this case we just want to return gracefully - # instead of spitting out an error. - rescue Octokit::NotFound - nil - end + # Yields the supplied block, responding to any rate limit errors. + # + # The exact strategy used for handling rate limiting errors depends on + # whether we are running in parallel mode or not. For more information see + # `#rate_or_wait_for_rate_limit`. + def with_rate_limit + request_count_counter.increment - def has_rate_limit? - return @has_rate_limit if defined?(@has_rate_limit) + raise_or_wait_for_rate_limit unless requests_remaining? - @has_rate_limit = rate_limit.present? - end + begin + yield + rescue Octokit::TooManyRequests + raise_or_wait_for_rate_limit - def rate_limit_exceed? - has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + # This retry will only happen when running in sequential mode as we'll + # raise an error in parallel mode. + retry + end end - def rate_limit_sleep_time - rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + # Returns `true` if we're still allowed to perform API calls. + def requests_remaining? + remaining_requests > RATE_LIMIT_THRESHOLD end - def request(method, *args, &block) - sleep rate_limit_sleep_time if rate_limit_exceed? - - data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend - return data unless data.is_a?(Array) + def remaining_requests + octokit.rate_limit.remaining + end - last_response = api.last_response + def raise_or_wait_for_rate_limit + rate_limit_counter.increment - if block_given? - yield data - # api.last_response could change while we're yielding (e.g. fetching labels for each PR) - # so we cache our own last response - each_response_page(last_response, &block) + if parallel? + raise RateLimitError else - each_response_page(last_response) { |page| data.concat(page) } - data + sleep(rate_limit_resets_in) end end - def each_response_page(last_response) - while last_response.rels[:next] - sleep rate_limit_sleep_time if rate_limit_exceed? - last_response = last_response.rels[:next].get - yield last_response.data if last_response.data.is_a?(Array) - end + def rate_limit_resets_in + # We add a few seconds to the rate limit so we don't _immediately_ + # resume when the rate limit resets as this may result in us performing + # a request before GitHub has a chance to reset the limit. + octokit.rate_limit.resets_in + 5 + end + + def respond_to_missing?(method, include_private = false) + octokit.respond_to?(method, include_private) + end + + def rate_limit_counter + @rate_limit_counter ||= Gitlab::Metrics.counter( + :github_importer_rate_limit_hits, + 'The number of times we hit the GitHub rate limit when importing projects' + ) + end + + def request_count_counter + @request_counter ||= Gitlab::Metrics.counter( + :github_importer_request_count, + 'The number of GitHub API calls performed when importing projects' + ) end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb new file mode 100644 index 00000000000..8274f37d358 --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (mr_id = find_merge_request_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: 'MergeRequest', + noteable_id: mr_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + commit_id: note.commit_id, + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: note.created_at, + updated_at: note.updated_at, + st_diff: note.diff_hash.to_yaml + } + + # It's possible that during an import we'll insert tens of thousands + # of diff notes. If we were to use the Note/LegacyDiffNote model here + # we'd also have to run additional queries for both validations and + # callbacks, putting a lot of pressure on the database. + # + # To work around this we're using bulk_insert with a single row. This + # allows us to efficiently insert data (even if it's just 1 row) + # without having to use all sorts of hacks to disable callbacks. + Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the merge request this note belongs to. + def find_merge_request_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb new file mode 100644 index 00000000000..966f12c5c2f --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNotesImporter + include ParallelScheduling + + def representation_class + Representation::DiffNote + end + + def importer_class + DiffNoteImporter + end + + def sidekiq_worker_class + ImportDiffNoteWorker + end + + def collection_method + :pull_requests_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb new file mode 100644 index 00000000000..bad064b76c8 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueAndLabelLinksImporter + attr_reader :issue, :project, :client + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + end + + def execute + IssueImporter.import_if_issue(issue, project, client) + LabelLinksImporter.new(issue, project, client).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb new file mode 100644 index 00000000000..31fefebf787 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueImporter + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, + :issuable_finder + + # Imports an issue if it's a regular issue and not a pull request. + def self.import_if_issue(issue, project, client) + new(issue, project, client).execute unless issue.pull_request? + end + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = GithubImport::IssuableFinder.new(project, issue) + end + + def execute + Issue.transaction do + if (issue_id = create_issue) + create_assignees(issue_id) + issuable_finder.cache_database_id(issue_id) + end + end + end + + # Creates a new GitLab issue for the current GitHub issue. + # + # Returns the ID of the created issue as an Integer. If the issue + # couldn't be created this method will return `nil` instead. + def create_issue + author_id, author_found = user_finder.author_id_for(issue) + + description = + MarkdownText.format(issue.description, issue.author, author_found) + + attributes = { + iid: issue.iid, + title: issue.truncated_title, + author_id: author_id, + project_id: project.id, + description: description, + milestone_id: milestone_finder.id_for(issue), + state: issue.state, + created_at: issue.created_at, + updated_at: issue.updated_at + } + + GithubImport.insert_and_return_id(attributes, project.issues) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the issue. + end + + # Stores all issue assignees in the database. + # + # issue_id - The ID of the created issue. + def create_assignees(issue_id) + assignees = [] + + issue.assignees.each do |assignee| + if (user_id = user_finder.user_id_for(assignee)) + assignees << { issue_id: issue_id, user_id: user_id } + end + end + + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb new file mode 100644 index 00000000000..ac6d0666b3a --- /dev/null +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssuesImporter + include ParallelScheduling + + def importer_class + IssueAndLabelLinksImporter + end + + def representation_class + Representation::Issue + end + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_imported_cache(issue) + issue.number + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb new file mode 100644 index 00000000000..2001b7e3482 --- /dev/null +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelLinksImporter + attr_reader :issue, :project, :client, :label_finder + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @label_finder = LabelFinder.new(project) + end + + def execute + create_labels + end + + def create_labels + time = Time.zone.now + rows = [] + target_id = find_target_id + + issue.label_names.each do |label_name| + # Although unlikely it's technically possible for an issue to be + # given a label that was created and assigned after we imported all + # the project's labels. + next unless (label_id = label_finder.id_for(label_name)) + + rows << { + label_id: label_id, + target_id: target_id, + target_type: issue.issuable_type, + created_at: time, + updated_at: time + } + end + + Gitlab::Database.bulk_insert(LabelLink.table_name, rows) + end + + def find_target_id + GithubImport::IssuableFinder.new(project, issue).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb new file mode 100644 index 00000000000..a73033d35ba --- /dev/null +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelsImporter + include BulkImporting + + attr_reader :project, :client, :existing_labels + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @client = client + @existing_labels = project.labels.pluck(:title).to_set + end + + def execute + bulk_insert(Label, build_labels) + build_labels_cache + end + + def build_labels + build_database_rows(each_label) + end + + def already_imported?(label) + existing_labels.include?(label.name) + end + + def build_labels_cache + LabelFinder.new(project).build_cache + end + + def build(label) + time = Time.zone.now + + { + title: label.name, + color: '#' + label.color, + project_id: project.id, + type: 'ProjectLabel', + created_at: time, + updated_at: time + } + end + + def each_label + client.labels(project.import_source) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb new file mode 100644 index 00000000000..c53480e828a --- /dev/null +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class MilestonesImporter + include BulkImporting + + attr_reader :project, :client, :existing_milestones + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_milestones = project.milestones.pluck(:iid).to_set + end + + def execute + bulk_insert(Milestone, build_milestones) + build_milestones_cache + end + + def build_milestones + build_database_rows(each_milestone) + end + + def already_imported?(milestone) + existing_milestones.include?(milestone.number) + end + + def build_milestones_cache + MilestoneFinder.new(project).build_cache + end + + def build(milestone) + { + iid: milestone.number, + title: milestone.title, + description: milestone.description, + project_id: project.id, + state: state_for(milestone), + created_at: milestone.created_at, + updated_at: milestone.updated_at + } + end + + def state_for(milestone) + milestone.state == 'open' ? :active : :closed + end + + def each_milestone + client.milestones(project.import_source, state: 'all') + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb new file mode 100644 index 00000000000..c890f2df360 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::Note`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (noteable_id = find_noteable_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: note.noteable_type, + noteable_id: noteable_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + created_at: note.created_at, + updated_at: note.updated_at + } + + # We're using bulk_insert here so we can bypass any validations and + # callbacks. Running these would result in a lot of unnecessary SQL + # queries being executed when importing large projects. + Gitlab::Database.bulk_insert(Note.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the issue or merge request to create the note for. + def find_noteable_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb new file mode 100644 index 00000000000..5aec760ea5f --- /dev/null +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NotesImporter + include ParallelScheduling + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def collection_method + :issues_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb new file mode 100644 index 00000000000..49d859f9624 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestImporter + attr_reader :pull_request, :project, :client, :user_finder, + :milestone_finder, :issuable_finder + + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(pull_request, project, client) + @pull_request = pull_request + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = + GithubImport::IssuableFinder.new(project, pull_request) + end + + def execute + if (mr_id = create_merge_request) + issuable_finder.cache_database_id(mr_id) + end + end + + # Creates the merge request and returns its ID. + # + # This method will return `nil` if the merge request could not be + # created. + def create_merge_request + author_id, author_found = user_finder.author_id_for(pull_request) + + description = MarkdownText + .format(pull_request.description, pull_request.author, author_found) + + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = GithubImport + .insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.find(merge_request_id) + + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_branch_sha = pull_request.target_branch_sha + + merge_request.keep_around_commit + merge_request.merge_request_diffs.create + + merge_request.id + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb new file mode 100644 index 00000000000..5437e32e9f1 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsImporter + include ParallelScheduling + + def importer_class + PullRequestImporter + end + + def representation_class + Representation::PullRequest + end + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def id_for_already_imported_cache(pr) + pr.number + end + + def each_object_to_import + super do |pr| + update_repository if update_repository?(pr) + yield pr + end + end + + def update_repository + # We set this column _before_ fetching the repository, and this is + # deliberate. If we were to update this column after the fetch we may + # miss out on changes pushed during the fetch or between the fetch and + # updating the timestamp. + project.update_column(:last_repository_updated_at, Time.zone.now) + + project.repository.fetch_remote('github', forced: false) + + pname = project.path_with_namespace + + Rails.logger + .info("GitHub importer finished updating repository for #{pname}") + + repository_updates_counter.increment(project: pname) + end + + def update_repository?(pr) + last_update = project.last_repository_updated_at || project.created_at + + return false if pr.updated_at < last_update + + # PRs may be updated without there actually being new commits, thus we + # check to make sure we only re-fetch if truly necessary. + !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + end + + def commit_exists?(sha) + project.repository.lookup(sha) + true + rescue Rugged::Error + false + end + + def collection_method + :pull_requests + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + def repository_updates_counter + @repository_updates_counter ||= Gitlab::Metrics.counter( + :github_importer_repository_updates, + 'The number of times repositories have to be updated again' + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb new file mode 100644 index 00000000000..100f459fdcc --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesImporter + include BulkImporting + + attr_reader :project, :client, :existing_tags + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_tags = project.releases.pluck(:tag).to_set + end + + def execute + bulk_insert(Release, build_releases) + end + + def build_releases + build_database_rows(each_release) + end + + def already_imported?(release) + existing_tags.include?(release.tag_name) + end + + def build(release) + { + tag: release.tag_name, + description: description_for(release), + created_at: release.created_at, + updated_at: release.updated_at, + project_id: project.id + } + end + + def each_release + client.releases(project.import_source) + end + + def description_for(release) + if release.body.present? + release.body + else + "Release for tag #{release.tag_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb new file mode 100644 index 00000000000..0b67fc8db73 --- /dev/null +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class RepositoryImporter + include Gitlab::ShellAdapter + + attr_reader :project, :client + + def initialize(project, client) + @project = project + @client = client + end + + # Returns true if we should import the wiki for the project. + def import_wiki? + client.repository(project.import_source)&.has_wiki && + !project.wiki_repository_exists? + end + + # Imports the repository data. + # + # This method will return true if the data was imported successfully or + # the repository had already been imported before. + def execute + imported = + # It's possible a repository has already been imported when running + # this code, e.g. because we had to retry this job after + # `import_wiki?` raised a rate limit error. In this case we'll skip + # re-importing the main repository. + if project.repository.empty_repo? + import_repository + else + true + end + + update_clone_time if imported + + imported = import_wiki_repository if import_wiki? && imported + + imported + end + + def import_repository + project.ensure_repository + + configure_repository_remote + + project.repository.fetch_remote('github', forced: true) + + true + rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + fail_import("Failed to import the repository: #{e.message}") + end + + def configure_repository_remote + return if project.repository.remote_exists?('github') + + project.repository.add_remote('github', project.import_url) + project.repository.set_import_remote_as_mirror('github') + + project.repository.add_remote_fetch_config( + 'github', + '+refs/pull/*/head:refs/merge-requests/*/head' + ) + end + + def import_wiki_repository + wiki_path = "#{project.disk_path}.wiki" + wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') + storage_path = project.repository_storage_path + + gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + + true + rescue Gitlab::Shell::Error => e + if e.message !~ /repository not exported/ + fail_import("Failed to import the wiki: #{e.message}") + else + true + end + end + + def update_clone_time + project.update_column(:last_repository_updated_at, Time.zone.now) + end + + def fail_import(message) + project.mark_import_as_failed(message) + false + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb new file mode 100644 index 00000000000..211915f1d87 --- /dev/null +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # IssuableFinder can be used for caching and retrieving database IDs for + # issuable objects such as issues and pull requests. By caching these IDs we + # remove the need for running a lot of database queries when importing + # GitHub projects. + class IssuableFinder + attr_reader :project, :object + + # The base cache key to use for storing/retrieving issuable IDs. + CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'.freeze + + # project - An instance of `Project`. + # object - The object to look up or set a database ID for. + def initialize(project, object) + @project = project + @object = object + end + + # Returns the database ID for the object. + # + # This method will return `nil` if no ID could be found. + def database_id + val = Caching.read(cache_key) + + val.to_i if val.present? + end + + # Associates the given database ID with the current object. + # + # database_id - The ID of the corresponding database row. + def cache_database_id(database_id) + Caching.write(cache_key, database_id) + end + + private + + def cache_key + CACHE_KEY % { + project: project.id, + type: cache_key_type, + iid: cache_key_iid + } + end + + # Returns the identifier to use for cache keys. + # + # For issues and pull requests this will be "Issue" or "MergeRequest" + # respectively. For diff notes this will return "MergeRequest", for + # regular notes it will either return "Issue" or "MergeRequest" depending + # on what type of object the note belongs to. + def cache_key_type + if object.respond_to?(:issuable_type) + object.issuable_type + elsif object.respond_to?(:noteable_type) + object.noteable_type + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + + def cache_key_iid + if object.respond_to?(:noteable_id) + object.noteable_id + elsif object.respond_to?(:iid) + object.iid + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb new file mode 100644 index 00000000000..9be071141db --- /dev/null +++ b/lib/gitlab/github_import/label_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class LabelFinder + attr_reader :project + + # The base cache key to use for storing/retrieving label IDs. + CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'.freeze + + # project - An instance of `Project`. + def initialize(project) + @project = project + end + + # Returns the label ID for the given name. + def id_for(name) + Caching.read_integer(cache_key_for(name)) + end + + def build_cache + mapping = @project + .labels + .pluck(:id, :name) + .each_with_object({}) do |(id, name), hash| + hash[cache_key_for(name)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(name) + CACHE_KEY % { project: project.id, name: name } + end + end + end +end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb new file mode 100644 index 00000000000..b25c4f7becf --- /dev/null +++ b/lib/gitlab/github_import/markdown_text.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MarkdownText + attr_reader :text, :author, :exists + + def self.format(*args) + new(*args).to_s + end + + # text - The Markdown text as a String. + # author - An instance of `Gitlab::GithubImport::Representation::User` + # exists - Boolean that indicates the user exists in the GitLab database. + def initialize(text, author, exists = false) + @text = text + @author = author + @exists = exists + end + + def to_s + if exists + text + else + "*Created by: #{author.login}*\n\n#{text}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb new file mode 100644 index 00000000000..208d15dc144 --- /dev/null +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MilestoneFinder + attr_reader :project + + # The base cache key to use for storing/retrieving milestone IDs. + CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'.freeze + + # project - An instance of `Project` + def initialize(project) + @project = project + end + + # issuable - An instance of `Gitlab::GithubImport::Representation::Issue` + # or `Gitlab::GithubImport::Representation::PullRequest`. + def id_for(issuable) + return unless issuable.milestone_number + + Caching.read_integer(cache_key_for(issuable.milestone_number)) + end + + def build_cache + mapping = @project + .milestones + .pluck(:id, :iid) + .each_with_object({}) do |(id, iid), hash| + hash[cache_key_for(iid)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(iid) + CACHE_KEY % { project: project.id, iid: iid } + end + end + end +end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb new file mode 100644 index 00000000000..c3db2d0b469 --- /dev/null +++ b/lib/gitlab/github_import/page_counter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # PageCounter can be used to keep track of the last imported page of a + # collection, allowing workers to resume where they left off in the event of + # an error. + class PageCounter + attr_reader :cache_key + + # The base cache key to use for storing the last page number. + CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}'.freeze + + def initialize(project, collection) + @cache_key = CACHE_KEY % { project: project.id, collection: collection } + end + + # Sets the page number to the given value. + # + # Returns true if the page number was overwritten, false otherwise. + def set(page) + Caching.write_if_greater(cache_key, page) + end + + # Returns the current value from the cache. + def current + Caching.read_integer(cache_key) || 1 + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb new file mode 100644 index 00000000000..81739834b41 --- /dev/null +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The ParallelImporter schedules the importing of a GitHub project using + # Sidekiq. + class ParallelImporter + attr_reader :project + + def self.async? + true + end + + def initialize(project) + @project = project + end + + def execute + jid = generate_jid + + # The original import JID is the JID of the RepositoryImportWorker job, + # which will be removed once that job completes. Reusing that JID could + # result in StuckImportJobsWorker marking the job as stuck before we get + # to running Stage::ImportRepositoryWorker. + # + # We work around this by setting the JID to a custom generated one, then + # refreshing it in the various stages whenever necessary. + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.update_column(:import_jid, jid) + + Stage::ImportRepositoryWorker + .perform_async(project.id) + + true + end + + def generate_jid + "github-importer/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb new file mode 100644 index 00000000000..d4d1357f5a3 --- /dev/null +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module ParallelScheduling + attr_reader :project, :client, :page_counter, :already_imported_cache_key + + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'github-importer/already-imported/%{project}/%{collection}'.freeze + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + # parallel - When set to true the objects will be imported in parallel. + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @page_counter = PageCounter.new(project, collection_method) + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def parallel? + @parallel + end + + def execute + retval = + if parallel? + parallel_import + else + sequential_import + end + + # Once we have completed all work we can remove our "already exists" + # cache so we don't put too much pressure on Redis. + # + # We don't immediately remove it since it's technically possible for + # other instances of this job to still run, instead we set the + # expiration time to a lower value. This prevents the other jobs from + # still scheduling duplicates while. Since all work has already been + # completed those jobs will just cycle through any remaining pages while + # not scheduling anything. + Caching.expire(already_imported_cache_key, 15.minutes.to_i) + + retval + end + + # Imports all the objects in sequence in the current thread. + def sequential_import + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + importer_class.new(repr, project, client).execute + end + end + + # Imports all objects in parallel by scheduling a Sidekiq job for every + # individual object. + def parallel_import + waiter = JobWaiter.new + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + sidekiq_worker_class + .perform_async(project.id, repr.to_hash, waiter.key) + + waiter.jobs_remaining += 1 + end + + waiter + end + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + # We inject the page number here to make sure that all importers always + # start where they left off. Simply starting over wouldn't work for + # repositories with a lot of data (e.g. tens of thousands of comments). + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, options) do |page| + # Technically it's possible that the same work is performed multiple + # times, as Sidekiq doesn't guarantee there will ever only be one + # instance of a job. In such a scenario it's possible for one job to + # have a lower page number (e.g. 5) compared to another (e.g. 10). In + # this case we skip over all the objects until we have caught up, + # reducing the number of duplicate jobs scheduled by the provided + # block. + next unless page_counter.set(page.number) + + page.objects.each do |object| + next if already_imported?(object) + + yield object + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(object) + end + end + end + + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(object) + id = id_for_already_imported_cache(object) + + Caching.set_includes?(already_imported_cache_key, id) + end + + # Marks the given object as "already imported". + def mark_as_imported(object) + id = id_for_already_imported_cache(object) + + Caching.set_add(already_imported_cache_key, id) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been imported or not. + # + # object - The object we may want to import. + def id_for_already_imported_cache(object) + raise NotImplementedError + end + + # The class used for converting API responses to Hashes when performing + # the import. + def representation_class + raise NotImplementedError + end + + # The class to use for importing objects when importing them sequentially. + def importer_class + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + # Any options to be passed to the method used for retrieving the data to + # import. + def collection_options + {} + end + end + end +end diff --git a/lib/gitlab/github_import/rate_limit_error.rb b/lib/gitlab/github_import/rate_limit_error.rb new file mode 100644 index 00000000000..cc2de909c29 --- /dev/null +++ b/lib/gitlab/github_import/rate_limit_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Error that will be raised when we're about to reach (or have reached) the + # GitHub API's rate limit. + RateLimitError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/github_import/representation.rb b/lib/gitlab/github_import/representation.rb new file mode 100644 index 00000000000..639477ef2a2 --- /dev/null +++ b/lib/gitlab/github_import/representation.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + TIMESTAMP_KEYS = %i[created_at updated_at merged_at].freeze + + # Converts a Hash with String based keys to one that can be used by the + # various Representation classes. + # + # Example: + # + # Representation.symbolize_hash('number' => 10) # => { number: 10 } + def self.symbolize_hash(raw_hash = nil) + hash = raw_hash.deep_symbolize_keys + + TIMESTAMP_KEYS.each do |key| + hash[key] = Time.parse(hash[key]) if hash[key].is_a?(String) + end + + hash + end + end + end +end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb new file mode 100644 index 00000000000..bb7439a0641 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class DiffNote + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, + :diff_hunk, :author, :note, :created_at, :updated_at, + :github_id + + NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + + # Builds a diff note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_ID_REGEX) + + unless matches + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: 'MergeRequest', + noteable_id: matches[:iid].to_i, + file_path: note.path, + commit_id: note.commit_id, + diff_hunk: note.diff_hunk, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + def line_code + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + + Gitlab::Git + .diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) + end + + # Returns a Hash that can be used to populate `notes.st_diff`, removing + # the need for requesting Git data for every diff note. + def diff_hash + { + diff: diff_hunk, + new_path: file_path, + old_path: file_path, + + # These fields are not displayed for LegacyDiffNote notes, so it + # doesn't really matter what we set them to. + a_mode: '100644', + b_mode: '100644', + new_file: false + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb new file mode 100644 index 00000000000..c3405759631 --- /dev/null +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ExposeAttribute + extend ActiveSupport::Concern + + module ClassMethods + # Defines getter methods for the given attribute names. + # + # Example: + # + # expose_attribute :iid, :title + def expose_attribute(*names) + names.each do |name| + name = name.to_sym + + define_method(name) { attributes[name] } + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb new file mode 100644 index 00000000000..f3071b3e2b3 --- /dev/null +++ b/lib/gitlab/github_import/representation/issue.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Issue + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :milestone_number, + :created_at, :updated_at, :state, :assignees, + :label_names, :author + + # Builds an issue from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the issue + # details. + def self.from_api_response(issue) + user = + if issue.user + Representation::User.from_api_response(issue.user) + end + + hash = { + iid: issue.number, + title: issue.title, + description: issue.body, + milestone_number: issue.milestone&.number, + state: issue.state == 'open' ? :opened : :closed, + assignees: issue.assignees.map do |u| + Representation::User.from_api_response(u) + end, + label_names: issue.labels.map(&:name), + author: user, + created_at: issue.created_at, + updated_at: issue.updated_at, + pull_request: issue.pull_request ? true : false + } + + new(hash) + end + + # Builds a new issue using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:assignees].map! { |u| Representation::User.from_json_hash(u) } + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A hash containing the raw issue details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + def labels? + label_names && label_names.any? + end + + def pull_request? + attributes[:pull_request] + end + + def issuable_type + pull_request? ? 'MergeRequest' : 'Issue' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb new file mode 100644 index 00000000000..a68bc4c002f --- /dev/null +++ b/lib/gitlab/github_import/representation/note.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Note + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_id, :noteable_type, :author, :note, + :created_at, :updated_at, :github_id + + NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + + # Builds a note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + + if !matches || !matches[:type] + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + noteable_type = + if matches[:type] == 'pull' + 'MergeRequest' + else + 'Issue' + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: noteable_type, + noteable_id: matches[:iid].to_i, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + alias_method :issuable_type, :noteable_type + end + end + end +end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb new file mode 100644 index 00000000000..593b491a837 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class PullRequest + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :source_branch, + :source_branch_sha, :target_branch, :target_branch_sha, + :milestone_number, :author, :assignee, :created_at, + :updated_at, :merged_at, :source_repository_id, + :target_repository_id, :source_repository_owner + + # Builds a PR from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the PR details. + def self.from_api_response(pr) + assignee = + if pr.assignee + Representation::User.from_api_response(pr.assignee) + end + + user = Representation::User.from_api_response(pr.user) if pr.user + hash = { + iid: pr.number, + title: pr.title, + description: pr.body, + source_branch: pr.head.ref, + target_branch: pr.base.ref, + source_branch_sha: pr.head.sha, + target_branch_sha: pr.base.sha, + source_repository_id: pr.head&.repo&.id, + target_repository_id: pr.base&.repo&.id, + source_repository_owner: pr.head&.user&.login, + state: pr.state == 'open' ? :opened : :closed, + milestone_number: pr.milestone&.number, + author: user, + assignee: assignee, + created_at: pr.created_at, + updated_at: pr.updated_at, + merged_at: pr.merged_at + } + + new(hash) + end + + # Builds a new PR using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + # Assignees are optional so we only convert it from a Hash if one was + # set. + hash[:assignee] &&= Representation::User + .from_json_hash(hash[:assignee]) + + new(hash) + end + + # attributes - A Hash containing the raw PR details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + # Returns a formatted source branch. + # + # For cross-project pull requests the branch name will be in the format + # `owner-name:branch-name`. + def formatted_source_branch + if cross_project? && source_repository_owner + "#{source_repository_owner}:#{source_branch}" + elsif source_branch == target_branch + # Sometimes the source and target branch are the same, but GitLab + # doesn't support this. This can happen when both the user and + # source repository have been deleted, and the PR was submitted from + # the fork's master branch. + "#{source_branch}-#{iid}" + else + source_branch + end + end + + def state + if merged_at + :merged + else + attributes[:state] + end + end + + def cross_project? + return true unless source_repository_id + + source_repository_id != target_repository_id + end + + def issuable_type + 'MergeRequest' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb new file mode 100644 index 00000000000..4a0f36ab8f0 --- /dev/null +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ToHash + # Converts the current representation to a Hash. The keys of this Hash + # will be Symbols. + def to_hash + hash = {} + + attributes.each do |key, value| + hash[key] = convert_value_for_to_hash(value) + end + + hash + end + + def convert_value_for_to_hash(value) + if value.is_a?(Array) + value.map { |v| convert_value_for_to_hash(v) } + elsif value.respond_to?(:to_hash) + value.to_hash + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb new file mode 100644 index 00000000000..e00dcfca33d --- /dev/null +++ b/lib/gitlab/github_import/representation/user.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class User + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login + + # Builds a user from a GitHub API response. + # + # user - An instance of `Sawyer::Resource` containing the user details. + def self.from_api_response(user) + new(id: user.id, login: user.login) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb new file mode 100644 index 00000000000..4f7324536a0 --- /dev/null +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The SequentialImporter imports a GitHub project in a single thread, + # without using Sidekiq. This makes it useful for testing purposes as well + # as Rake tasks, but it should be avoided for anything else in favour of the + # parallel importer. + class SequentialImporter + attr_reader :project, :client + + SEQUENTIAL_IMPORTERS = [ + Importer::LabelsImporter, + Importer::MilestonesImporter, + Importer::ReleasesImporter + ].freeze + + PARALLEL_IMPORTERS = [ + Importer::PullRequestsImporter, + Importer::IssuesImporter, + Importer::DiffNotesImporter, + Importer::NotesImporter + ].freeze + + # project - The project to import the data into. + # token - The token to use for the GitHub API. + def initialize(project, token: nil) + @project = project + @client = GithubImport + .new_client_for(project, token: token, parallel: false) + end + + def execute + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + PARALLEL_IMPORTERS.each do |klass| + klass.new(project, client, parallel: false).execute + end + + project.repository.after_import + + true + end + end + end +end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb new file mode 100644 index 00000000000..be1259662a7 --- /dev/null +++ b/lib/gitlab/github_import/user_finder.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Class that can be used for finding a GitLab user ID based on a GitHub user + # ID or username. + # + # Any found user IDs are cached in Redis to reduce the number of SQL queries + # executed over time. Valid keys are refreshed upon access so frequently + # used keys stick around. + # + # Lookups are cached even if no ID was found to remove the need for querying + # the database when most queries are not going to return results anyway. + class UserFinder + attr_reader :project, :client + + # The base cache key to use for caching user IDs for a given GitHub user + # ID. + ID_CACHE_KEY = 'github-import/user-finder/user-id/%s'.freeze + + # The base cache key to use for caching user IDs for a given GitHub email + # address. + ID_FOR_EMAIL_CACHE_KEY = + 'github-import/user-finder/id-for-email/%s'.freeze + + # The base cache key to use for caching the Email addresses of GitHub + # usernames. + EMAIL_FOR_USERNAME_CACHE_KEY = + 'github-import/user-finder/email-for-username/%s'.freeze + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + end + + # Returns the GitLab user ID of an object's author. + # + # If the object has no author ID we'll use the ID of the GitLab ghost + # user. + def author_id_for(object) + id = + if object&.author + user_id_for(object.author) + else + GithubImport.ghost_user_id + end + + if id + [id, true] + else + [project.creator_id, false] + end + end + + # Returns the GitLab user ID of an issuable's assignee. + def assignee_id_for(issuable) + user_id_for(issuable.assignee) if issuable.assignee + end + + # Returns the GitLab user ID for a GitHub user. + # + # user - An instance of `Gitlab::GithubImport::Representation::User`. + def user_id_for(user) + find(user.id, user.login) + end + + # Returns the GitLab ID for the given GitHub ID or username. + # + # id - The ID of the GitHub user. + # username - The username of the GitHub user. + def find(id, username) + email = email_for_github_username(username) + cached, found_id = find_from_cache(id, email) + + return found_id if found_id + + # We only want to query the database if necessary. If previous lookups + # didn't yield a user ID we won't query the database again until the + # keys expire. + find_id_from_database(id, email) unless cached + end + + # Finds a user ID from the cache for a given GitHub ID or Email. + def find_from_cache(id, email = nil) + id_exists, id_for_github_id = cached_id_for_github_id(id) + + return [id_exists, id_for_github_id] if id_for_github_id + + # Just in case no Email address could be retrieved (for whatever reason) + return [false] unless email + + cached_id_for_github_email(email) + end + + # Finds a GitLab user ID from the database for a given GitHub user ID or + # Email. + def find_id_from_database(id, email) + id_for_github_id(id) || id_for_github_email(email) + end + + def email_for_github_username(username) + cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username + email = Caching.read(cache_key) + + unless email + user = client.user(username) + email = Caching.write(cache_key, user.email) if user + end + + email + end + + def cached_id_for_github_id(id) + read_id_from_cache(ID_CACHE_KEY % id) + end + + def cached_id_for_github_email(email) + read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email) + end + + # Queries and caches the GitLab user ID for a GitHub user ID, if one was + # found. + def id_for_github_id(id) + gitlab_id = query_id_for_github_id(id) || nil + + Caching.write(ID_CACHE_KEY % id, gitlab_id) + end + + # Queries and caches the GitLab user ID for a GitHub email, if one was + # found. + def id_for_github_email(email) + gitlab_id = query_id_for_github_email(email) || nil + + Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) + end + + def query_id_for_github_id(id) + User.for_github_id(id).pluck(:id).first + end + + def query_id_for_github_email(email) + User.by_any_email(email).pluck(:id).first + end + + # Reads an ID from the cache. + # + # The return value is an Array with two values: + # + # 1. A boolean indicating if the key was present or not. + # 2. The ID as an Integer, or nil in case no ID could be found. + def read_id_from_cache(key) + value = Caching.read(key) + exists = !value.nil? + number = value.to_i + + # The cache key may be empty to indicate a previously looked up user for + # which we couldn't find an ID. + [exists, number.positive? ? number : nil] + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 239dca4d43f..263599831bf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -54,7 +54,6 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules - - :cluster - :services - :hooks - protected_branches: diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 81a213e8321..61db4bd9ccc 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -26,7 +26,7 @@ module Gitlab end def fetch_ref - @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch) + @project.repository.fetch_ref(@project.repository, source_ref: @diff_head_sha, target_ref: @merge_request.source_branch) end def branch_exists?(branch_name) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 0d745e2b21f..2b34ceb5831 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,8 +8,6 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', - cluster: 'Clusters::Cluster', - clusters: 'Clusters::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5404dc11a87..eeb03625479 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,14 +8,14 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Github::Import), + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) ].freeze class << self diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 4d6bbda15f3..f654508c391 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,11 +19,13 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end - attr_reader :key, :jobs_remaining, :finished + attr_reader :key, :finished + attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for - def initialize(jobs_remaining) - @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + # key - The key of this waiter. + def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + @key = key @jobs_remaining = jobs_remaining @finished = [] end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb new file mode 100644 index 00000000000..7a50f07f3c5 --- /dev/null +++ b/lib/gitlab/kubernetes/helm.rb @@ -0,0 +1,96 @@ +module Gitlab + module Kubernetes + class Helm + HELM_VERSION = '2.7.0'.freeze + NAMESPACE = 'gitlab-managed-apps'.freeze + INSTALL_DEPS = <<-EOS.freeze + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + EOS + + InstallCommand = Struct.new(:name, :install_helm, :chart) do + def pod_name + "install-#{name}" + end + end + + def initialize(kubeclient) + @kubeclient = kubeclient + @namespace = Namespace.new(NAMESPACE, kubeclient) + end + + def install(command) + @namespace.ensure_exists! + @kubeclient.create_pod(pod_resource(command)) + end + + ## + # Returns Pod phase + # + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + # + # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" + # + def installation_status(pod_name) + @kubeclient.get_pod(pod_name, @namespace.name).status.phase + end + + def installation_log(pod_name) + @kubeclient.get_pod_log(pod_name, @namespace.name).body + end + + def delete_installation_pod!(pod_name) + @kubeclient.delete_pod(pod_name, @namespace.name) + end + + private + + def pod_resource(command) + labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } + metadata = { name: command.pod_name, namespace: @namespace.name, labels: labels } + container = { + name: 'helm', + image: 'alpine:3.6', + env: generate_pod_env(command), + command: %w(/bin/sh), + args: %w(-c $(COMMAND_SCRIPT)) + } + spec = { containers: [container], restartPolicy: 'Never' } + + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) + end + + def generate_pod_env(command) + { + HELM_VERSION: HELM_VERSION, + TILLER_NAMESPACE: @namespace.name, + COMMAND_SCRIPT: generate_script(command) + }.map { |key, value| { name: key, value: value } } + end + + def generate_script(command) + [ + INSTALL_DEPS, + helm_init_command(command), + helm_install_command(command) + ].join("\n") + end + + def helm_init_command(command) + if command.install_helm + 'helm init >/dev/null' + else + 'helm init --client-only >/dev/null' + end + end + + def helm_install_command(command) + return if command.chart.nil? + + "helm install #{command.chart} --name #{command.name} --namespace #{@namespace.name} >/dev/null" + end + end + end +end diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb new file mode 100644 index 00000000000..c8479fbc0e8 --- /dev/null +++ b/lib/gitlab/kubernetes/namespace.rb @@ -0,0 +1,29 @@ +module Gitlab + module Kubernetes + class Namespace + attr_accessor :name + + def initialize(name, client) + @name = name + @client = client + end + + def exists? + @client.get_namespace(name) + rescue ::KubeException => ke + raise ke unless ke.error_code == 404 + false + end + + def create! + resource = ::Kubeclient::Resource.new(metadata: { name: name }) + + @client.create_namespace(resource) + end + + def ensure_exists! + exists? || create! + end + end + end +end diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb new file mode 100644 index 00000000000..f3842cdf762 --- /dev/null +++ b/lib/gitlab/kubernetes/pod.rb @@ -0,0 +1,12 @@ +module Gitlab + module Kubernetes + module Pod + PENDING = 'Pending'.freeze + RUNNING = 'Running'.freeze + SUCCEEDED = 'Succeeded'.freeze + FAILED = 'Failed'.freeze + UNKNOWN = 'Unknown'.freeze + PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index f330041cc00..2f07fde406c 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BaseFormatter attr_reader :client, :formatter, :project, :raw_data diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 8aa885fb811..80fe1d67209 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BranchFormatter < BaseFormatter delegate :repo, :sha, :ref, to: :raw_data diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb new file mode 100644 index 00000000000..53c910d44bd --- /dev/null +++ b/lib/gitlab/legacy_github_import/client.rb @@ -0,0 +1,148 @@ +module Gitlab + module LegacyGithubImport + class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + + attr_reader :access_token, :host, :api_version + + def initialize(access_token, host: nil, api_version: 'v3') + @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version + @users = {} + + if access_token + ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: api_endpoint, + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' + end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "repo, user, user:email" + }) + end + + def get_token(code) + client.auth_code.get_token(code).token + end + + def method_missing(method, *args, &block) + if api.respond_to?(method) + request(method, *args, &block) + else + super(method, *args, &block) + end + end + + def respond_to?(method) + api.respond_to?(method) || super + end + + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + + private + + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + end + + def github_options + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end + end + + def rate_limit + api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? + end + + def rate_limit_exceed? + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request(method, *args, &block) + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend + return data unless data.is_a?(Array) + + last_response = api.last_response + + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + yield last_response.data if last_response.data.is_a?(Array) + end + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index 8911b81ec9a..d2c7a8ae9f4 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class CommentFormatter < BaseFormatter attr_writer :author_id diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b8c07460ebb..12c968805f5 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class Importer include Gitlab::ShellAdapter diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index 27b171d6ddb..de55382d3ad 100644 --- a/lib/gitlab/github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssuableFormatter < BaseFormatter attr_writer :assignee_id, :author_id diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 977cd0423ba..4c8825ccf19 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssueFormatter < IssuableFormatter def attributes { diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 211ccdc51bb..c3eed12e739 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class LabelFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index dd782eff059..a565294384d 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class MilestoneFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index a55adc9b1c8..41e7eac4d08 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ProjectCreator include Gitlab::CurrentSettings diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 150afa31432..94c2e99066a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class PullRequestFormatter < IssuableFormatter delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 1ad702a6058..3ed9d4f76da 100644 --- a/lib/gitlab/github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ReleaseFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 04c2964da20..6d8055622f1 100644 --- a/lib/gitlab/github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class UserFormatter attr_reader :client, :raw diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index ca8d96f5650..27f45875c7c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class WikiFormatter attr_reader :project diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index d01de5eef0a..5919ebb1493 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -6,8 +6,6 @@ module Gitlab @worker_class = worker_class end - protected - def labels { controller: @worker_class.name, action: 'perform' } end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 8853dfa3d2d..5e4932e4e57 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -66,11 +66,7 @@ module Gitlab end def whitelisted_routes - logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def logout_route - route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end def sidekiq_route diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 22f8dd669d0..9a91f8bf96a 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -112,22 +112,6 @@ module Gitlab # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ - - activity - analytics - audit_events - avatar - edit - group_members - hooks - issues - labels - ldap - ldap_group_links - merge_requests - milestones - notification_setting - pipeline_quota - projects ].freeze ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index e57890f1143..910533076b0 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -40,5 +40,24 @@ module Gitlab def self.url_helpers @url_helpers ||= Gitlab::Application.routes.url_helpers end + + def self.redirect_legacy_paths(router, *paths) + build_redirect_path = lambda do |request, _params, path| + # Only replace the last occurence of `path`. + # + # `request.fullpath` includes the querystring + path = request.path.sub(%r{/#{path}/*(?!.*#{path})}, "/-/#{path}/") + path << "?#{request.query_string}" if request.query_string.present? + + path + end + + paths.each do |path| + router.match "/#{path}(/*rest)", + via: [:get, :post, :patch, :delete], + to: router.redirect { |params, request| build_redirect_path.call(request, params, path) }, + as: "legacy_#{path}_redirect" + end + end end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 1650263b98d..9dcf44fdc3e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,24 +33,29 @@ namespace :gitlab do backup.unpack unless backup.skipped?('db') - unless ENV['force'] == 'yes' - warning = <<-MSG.strip_heredoc - Before restoring the database we recommend removing all existing - tables to avoid future upgrade problems. Be aware that if you have - custom tables in the GitLab database these tables and all data will be - removed. - MSG - puts warning.color(:red) - ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) + begin + unless ENV['force'] == 'yes' + warning = <<-MSG.strip_heredoc + Before restoring the database, we will remove all existing + tables to avoid future upgrade problems. Be aware that if you have + custom tables in the GitLab database these tables and all data will be + removed. + MSG + puts warning.color(:red) + ask_to_continue + puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + $progress.puts 'Cleaning the database ... '.color(:blue) + Rake::Task['gitlab:db:drop_tables'].invoke + $progress.puts 'done'.color(:green) + Rake::Task['gitlab:backup:db:restore'].invoke + rescue Gitlab::TaskAbortedByUserError + puts "Quitting...".color(:red) + exit 1 end - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - $progress.puts 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - $progress.puts 'done'.color(:green) - Rake::Task['gitlab:backup:db:restore'].invoke end Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index d227a0c8bdb..adfcc3cda22 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,23 +2,21 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) - # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production + # 1. copy the bare repos to a specific path that contain the group or subgroups structure as folders + # 2. run: bundle exec rake gitlab:import:repos[/path/to/repos] RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped - # - # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" - task repos: :environment do - if Project.current_application_settings.hashed_storage_enabled - puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) + task :repos, [:import_path] => :environment do |_t, args| + unless args.import_path + puts 'Please specify an import path that contains the repositories'.color(:red) exit 1 end - Gitlab::BareRepositoryImporter.execute + Gitlab::BareRepositoryImport::Importer.execute(args.import_path) end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 7f86fd7b45e..8e9aef49e89 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,7 +7,7 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { token: token, verbose: true } + @options = { token: token } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first @@ -42,7 +42,9 @@ class GithubImport import_success = false timings = Benchmark.measure do - import_success = Github::Import.new(@project, @options).execute + import_success = Gitlab::GithubImport::SequentialImporter + .new(@project, token: @options[:token]) + .execute end if import_success diff --git a/package.json b/package.json index e607981143d..6315f9eced9 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,8 @@ "three-orbit-controls": "^82.1.0", "three-stl-loader": "^1.0.4", "timeago.js": "^2.0.5", + "ts-loader": "^3.1.1", + "typescript": "^2.6.1", "underscore": "^1.8.3", "url-loader": "^0.5.8", "visibilityjs": "^1.2.4", diff --git a/qa/qa/scenario/test/integration/mattermost.rb b/qa/qa/scenario/test/integration/mattermost.rb index 59d7dcd3d23..0220da929ce 100644 --- a/qa/qa/scenario/test/integration/mattermost.rb +++ b/qa/qa/scenario/test/integration/mattermost.rb @@ -7,11 +7,11 @@ module QA # including staging and on-premises installation. # class Mattermost < Scenario::Entrypoint - tags :mattermost + tags :core, :mattermost def perform(address, mattermost, *files) Runtime::Scenario.mattermost = mattermost - super(address, files) + super(address, *files) end end end diff --git a/spec/controllers/concerns/issuable_collections_spec.rb b/spec/controllers/concerns/issuable_collections_spec.rb index c9687af4dd2..cd3bf785d34 100644 --- a/spec/controllers/concerns/issuable_collections_spec.rb +++ b/spec/controllers/concerns/issuable_collections_spec.rb @@ -17,60 +17,6 @@ describe IssuableCollections do controller end - describe '#redirect_out_of_range' do - before do - allow(controller).to receive(:url_for) - end - - it 'returns true and redirects if the offset is out of range' do - relation = double(:relation, current_page: 10) - - expect(controller).to receive(:redirect_to) - expect(controller.send(:redirect_out_of_range, relation, 2)).to eq(true) - end - - it 'returns false if the offset is not out of range' do - relation = double(:relation, current_page: 1) - - expect(controller).not_to receive(:redirect_to) - expect(controller.send(:redirect_out_of_range, relation, 2)).to eq(false) - end - end - - describe '#issues_page_count' do - it 'returns the number of issue pages' do - project = create(:project, :public) - - create(:issue, project: project) - - finder = IssuesFinder.new(user) - issues = finder.execute - - allow(controller).to receive(:issues_finder) - .and_return(finder) - - expect(controller.send(:issues_page_count, issues)).to eq(1) - end - end - - describe '#merge_requests_page_count' do - it 'returns the number of merge request pages' do - project = create(:project, :public) - - create(:merge_request, source_project: project, target_project: project) - - finder = MergeRequestsFinder.new(user) - merge_requests = finder.execute - - allow(controller).to receive(:merge_requests_finder) - .and_return(finder) - - pages = controller.send(:merge_requests_page_count, merge_requests) - - expect(pages).to eq(1) - end - end - describe '#page_count_for_relation' do it 'returns the number of pages' do relation = double(:relation, limit_value: 20) diff --git a/spec/controllers/import/github_controller_spec.rb b/spec/controllers/import/github_controller_spec.rb index 45c3fa075ef..9bbd97ec305 100644 --- a/spec/controllers/import/github_controller_spec.rb +++ b/spec/controllers/import/github_controller_spec.rb @@ -21,9 +21,9 @@ describe Import::GithubController do describe "GET callback" do it "updates access token" do token = "asdasd12345" - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:get_token).and_return(token) - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:github_options).and_return({}) stub_omniauth_provider('github') diff --git a/spec/controllers/projects/clusters/applications_controller_spec.rb b/spec/controllers/projects/clusters/applications_controller_spec.rb new file mode 100644 index 00000000000..8b460646059 --- /dev/null +++ b/spec/controllers/projects/clusters/applications_controller_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe Projects::Clusters::ApplicationsController do + include AccessMatchersForController + + def current_application + Clusters::Cluster::APPLICATIONS[application] + end + + describe 'POST create' do + let(:cluster) { create(:cluster, :project, :provided_by_gcp) } + let(:project) { cluster.project } + let(:application) { 'helm' } + let(:params) { { application: application, id: cluster.id } } + + describe 'functionality' do + let(:user) { create(:user) } + + before do + project.add_master(user) + sign_in(user) + end + + it 'schedule an application installation' do + expect(ClusterInstallAppWorker).to receive(:perform_async).with(application, anything).once + + expect { go }.to change { current_application.count } + expect(response).to have_http_status(:no_content) + expect(cluster.application_helm).to be_scheduled + end + + context 'when cluster do not exists' do + before do + cluster.destroy! + end + + it 'return 404' do + expect { go }.not_to change { current_application.count } + expect(response).to have_http_status(:not_found) + end + end + + context 'when application is unknown' do + let(:application) { 'unkwnown-app' } + + it 'return 404' do + go + + expect(response).to have_http_status(:not_found) + end + end + + context 'when application is already installing' do + before do + create(:cluster_applications_helm, :installing, cluster: cluster) + end + + it 'returns 400' do + go + + expect(response).to have_http_status(:bad_request) + end + end + end + + describe 'security' do + before do + allow(ClusterInstallAppWorker).to receive(:perform_async) + end + + it { expect { go }.to be_allowed_for(:admin) } + it { expect { go }.to be_allowed_for(:owner).of(project) } + it { expect { go }.to be_allowed_for(:master).of(project) } + it { expect { go }.to be_denied_for(:developer).of(project) } + it { expect { go }.to be_denied_for(:reporter).of(project) } + it { expect { go }.to be_denied_for(:guest).of(project) } + it { expect { go }.to be_denied_for(:user) } + it { expect { go }.to be_denied_for(:external) } + end + + def go + post :create, params.merge(namespace_id: project.namespace, project_id: project) + end + end +end diff --git a/spec/controllers/projects/commit_controller_spec.rb b/spec/controllers/projects/commit_controller_spec.rb index 4612fc6e441..5dc27e2bbba 100644 --- a/spec/controllers/projects/commit_controller_spec.rb +++ b/spec/controllers/projects/commit_controller_spec.rb @@ -134,8 +134,8 @@ describe Projects::CommitController do end end - describe "GET branches" do - it "contains branch and tags information" do + describe 'GET branches' do + it 'contains branch and tags information' do commit = project.commit('5937ac0a7beb003549fc5fd26fc247adbce4a52e') get(:branches, @@ -143,8 +143,26 @@ describe Projects::CommitController do project_id: project, id: commit.id) - expect(assigns(:branches)).to include("master", "feature_conflict") - expect(assigns(:tags)).to include("v1.1.0") + expect(assigns(:branches)).to include('master', 'feature_conflict') + expect(assigns(:branches_limit_exceeded)).to be_falsey + expect(assigns(:tags)).to include('v1.1.0') + expect(assigns(:tags_limit_exceeded)).to be_falsey + end + + it 'returns :limit_exceeded when number of branches/tags reach a threshhold' do + commit = project.commit('5937ac0a7beb003549fc5fd26fc247adbce4a52e') + allow_any_instance_of(Repository).to receive(:branch_count).and_return(1001) + allow_any_instance_of(Repository).to receive(:tag_count).and_return(1001) + + get(:branches, + namespace_id: project.namespace, + project_id: project, + id: commit.id) + + expect(assigns(:branches)).to eq([]) + expect(assigns(:branches_limit_exceeded)).to be_truthy + expect(assigns(:tags)).to eq([]) + expect(assigns(:tags_limit_exceeded)).to be_truthy end end diff --git a/spec/factories/clusters/applications/helm.rb b/spec/factories/clusters/applications/helm.rb new file mode 100644 index 00000000000..fab37195113 --- /dev/null +++ b/spec/factories/clusters/applications/helm.rb @@ -0,0 +1,35 @@ +FactoryGirl.define do + factory :cluster_applications_helm, class: Clusters::Applications::Helm do + cluster factory: %i(cluster provided_by_gcp) + + trait :not_installable do + status(-2) + end + + trait :installable do + status 0 + end + + trait :scheduled do + status 1 + end + + trait :installing do + status 2 + end + + trait :installed do + status 3 + end + + trait :errored do + status(-1) + status_reason 'something went wrong' + end + + trait :timeouted do + installing + updated_at ClusterWaitForAppInstallationWorker::TIMEOUT.ago + end + end +end diff --git a/spec/factories/clusters/applications/ingress.rb b/spec/factories/clusters/applications/ingress.rb new file mode 100644 index 00000000000..b103a980655 --- /dev/null +++ b/spec/factories/clusters/applications/ingress.rb @@ -0,0 +1,35 @@ +FactoryGirl.define do + factory :cluster_applications_ingress, class: Clusters::Applications::Ingress do + cluster factory: %i(cluster provided_by_gcp) + + trait :not_installable do + status(-2) + end + + trait :installable do + status 0 + end + + trait :scheduled do + status 1 + end + + trait :installing do + status 2 + end + + trait :installed do + status 3 + end + + trait :errored do + status(-1) + status_reason 'something went wrong' + end + + trait :timeouted do + installing + updated_at ClusterWaitForAppInstallationWorker::TIMEOUT.ago + end + end +end diff --git a/spec/features/projects/clusters_spec.rb b/spec/features/projects/clusters_spec.rb index 368aad860e7..197e6df4997 100644 --- a/spec/features/projects/clusters_spec.rb +++ b/spec/features/projects/clusters_spec.rb @@ -50,10 +50,24 @@ feature 'Clusters', :js do it 'user sees a cluster details page and creation status' do expect(page).to have_content('Cluster is being created on Google Container Engine...') + # Application Installation buttons + page.within('.js-cluster-application-row-helm') do + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button').text).to eq('Install') + end + Clusters::Cluster.last.provider.make_created! expect(page).to have_content('Cluster was successfully created on Google Container Engine') end + + it 'user sees a error if something worng during creation' do + expect(page).to have_content('Cluster is being created on Google Container Engine...') + + Clusters::Cluster.last.provider.make_errored!('Something wrong!') + + expect(page).to have_content('Something wrong!') + end end context 'when user filled form with invalid parameters' do @@ -78,6 +92,78 @@ feature 'Clusters', :js do it 'user sees an cluster details page' do expect(page).to have_button('Save') expect(page.find(:css, '.cluster-name').value).to eq(cluster.name) + + # Application Installation buttons + page.within('.js-cluster-application-row-helm') do + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to be_nil + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + end + end + + context 'when user installs application: Helm Tiller' do + before do + allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) + + page.within('.js-cluster-application-row-helm') do + page.find(:css, '.js-cluster-application-install-button').click + end + end + + it 'user sees status transition' do + page.within('.js-cluster-application-row-helm') do + # FE sends request and gets the response, then the buttons is "Install" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + + Clusters::Cluster.last.application_helm.make_installing! + + # FE starts polling and update the buttons to "Installing" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') + + Clusters::Cluster.last.application_helm.make_installed! + + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') + end + + expect(page).to have_content('Helm Tiller was successfully installed on your cluster') + end + end + + context 'when user installs application: Ingress' do + before do + allow(ClusterInstallAppWorker).to receive(:perform_async).and_return(nil) + # Helm Tiller needs to be installed before you can install Ingress + create(:cluster_applications_helm, :installed, cluster: cluster) + + visit project_clusters_path(project) + + page.within('.js-cluster-application-row-ingress') do + page.find(:css, '.js-cluster-application-install-button').click + end + end + + it 'user sees status transition' do + page.within('.js-cluster-application-row-ingress') do + # FE sends request and gets the response, then the buttons is "Install" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Install') + + Clusters::Cluster.last.application_ingress.make_installing! + + # FE starts polling and update the buttons to "Installing" + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installing') + + Clusters::Cluster.last.application_ingress.make_installed! + + expect(page.find(:css, '.js-cluster-application-install-button')['disabled']).to eq('true') + expect(page.find(:css, '.js-cluster-application-install-button')).to have_content('Installed') + end + + expect(page).to have_content('Ingress was successfully installed on your cluster') + end end context 'when user disables the cluster' do diff --git a/spec/features/projects/commit/mini_pipeline_graph_spec.rb b/spec/features/projects/commit/mini_pipeline_graph_spec.rb index 807a2189cc4..91282063a8d 100644 --- a/spec/features/projects/commit/mini_pipeline_graph_spec.rb +++ b/spec/features/projects/commit/mini_pipeline_graph_spec.rb @@ -12,6 +12,13 @@ feature 'Mini Pipeline Graph in Commit View', :js do end let(:build) { create(:ci_build, pipeline: pipeline) } + it 'display icon with status' do + build.run + visit project_commit_path(project, project.commit.id) + + expect(page).to have_selector('.ci-status-icon-running') + end + it 'displays a mini pipeline graph' do build.run visit project_commit_path(project, project.commit.id) diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz Binary files differindex 9614c72cdc3..fb6a3b8e733 100644 --- a/spec/features/projects/import_export/test_project_export.tar.gz +++ b/spec/features/projects/import_export/test_project_export.tar.gz diff --git a/spec/fixtures/api/schemas/cluster_status.json b/spec/fixtures/api/schemas/cluster_status.json index 1f255a17881..489d563be2b 100644 --- a/spec/fixtures/api/schemas/cluster_status.json +++ b/spec/fixtures/api/schemas/cluster_status.json @@ -1,11 +1,38 @@ { "type": "object", "required" : [ - "status" + "status", + "applications" ], "properties" : { "status": { "type": "string" }, - "status_reason": { "type": ["string", "null"] } + "status_reason": { "type": ["string", "null"] }, + "applications": { + "type": "array", + "items": { "$ref": "#/definitions/application_status" } + } }, - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "application_status": { + "type": "object", + "additionalProperties": false, + "properties" : { + "name": { "type": "string" }, + "status": { + "type": { + "enum": [ + "installable", + "scheduled", + "installing", + "installed", + "errored" + ] + } + }, + "status_reason": { "type": ["string", "null"] } + }, + "required" : [ "name", "status" ] + } + } } diff --git a/spec/helpers/labels_helper_spec.rb b/spec/helpers/labels_helper_spec.rb index 36d6e495ed0..4ac4302adfd 100644 --- a/spec/helpers/labels_helper_spec.rb +++ b/spec/helpers/labels_helper_spec.rb @@ -24,7 +24,7 @@ describe LabelsHelper do let(:group) { build(:group, name: 'bar') } it 'links to group issues page' do - expect(link_to_label(label, subject: group)).to match %r{<a href="/groups/bar/issues\?label_name%5B%5D=#{label.name}">.*</a>} + expect(link_to_label(label, subject: group)).to match %r{<a href="/groups/bar/-/issues\?label_name%5B%5D=#{label.name}">.*</a>} end end diff --git a/spec/javascripts/clusters/clusters_bundle_spec.js b/spec/javascripts/clusters/clusters_bundle_spec.js new file mode 100644 index 00000000000..027e8001053 --- /dev/null +++ b/spec/javascripts/clusters/clusters_bundle_spec.js @@ -0,0 +1,257 @@ +import Clusters from '~/clusters/clusters_bundle'; +import { + APPLICATION_INSTALLABLE, + APPLICATION_INSTALLING, + APPLICATION_INSTALLED, + REQUEST_LOADING, + REQUEST_SUCCESS, + REQUEST_FAILURE, +} from '~/clusters/constants'; +import getSetTimeoutPromise from '../helpers/set_timeout_promise_helper'; + +describe('Clusters', () => { + let cluster; + preloadFixtures('clusters/show_cluster.html.raw'); + + beforeEach(() => { + loadFixtures('clusters/show_cluster.html.raw'); + cluster = new Clusters(); + }); + + afterEach(() => { + cluster.destroy(); + }); + + describe('toggle', () => { + it('should update the button and the input field on click', () => { + cluster.toggleButton.click(); + + expect( + cluster.toggleButton.classList, + ).not.toContain('checked'); + + expect( + cluster.toggleInput.getAttribute('value'), + ).toEqual('false'); + }); + }); + + describe('checkForNewInstalls', () => { + const INITIAL_APP_MAP = { + helm: { status: null, title: 'Helm Tiller' }, + ingress: { status: null, title: 'Ingress' }, + runner: { status: null, title: 'GitLab Runner' }, + }; + + it('does not show alert when things transition from initial null state to something', () => { + cluster.checkForNewInstalls(INITIAL_APP_MAP, { + ...INITIAL_APP_MAP, + helm: { status: APPLICATION_INSTALLABLE, title: 'Helm Tiller' }, + }); + + expect(document.querySelector('.js-cluster-application-notice .flash-text')).toBeNull(); + }); + + it('shows an alert when something gets newly installed', () => { + cluster.checkForNewInstalls({ + ...INITIAL_APP_MAP, + helm: { status: APPLICATION_INSTALLING, title: 'Helm Tiller' }, + }, { + ...INITIAL_APP_MAP, + helm: { status: APPLICATION_INSTALLED, title: 'Helm Tiller' }, + }); + + expect(document.querySelector('.js-cluster-application-notice .flash-text')).toBeDefined(); + expect(document.querySelector('.js-cluster-application-notice .flash-text').textContent.trim()).toEqual('Helm Tiller was successfully installed on your cluster'); + }); + + it('shows an alert when multiple things gets newly installed', () => { + cluster.checkForNewInstalls({ + ...INITIAL_APP_MAP, + helm: { status: APPLICATION_INSTALLING, title: 'Helm Tiller' }, + ingress: { status: APPLICATION_INSTALLABLE, title: 'Ingress' }, + }, { + ...INITIAL_APP_MAP, + helm: { status: APPLICATION_INSTALLED, title: 'Helm Tiller' }, + ingress: { status: APPLICATION_INSTALLED, title: 'Ingress' }, + }); + + expect(document.querySelector('.js-cluster-application-notice .flash-text')).toBeDefined(); + expect(document.querySelector('.js-cluster-application-notice .flash-text').textContent.trim()).toEqual('Helm Tiller, Ingress was successfully installed on your cluster'); + }); + }); + + describe('updateContainer', () => { + describe('when creating cluster', () => { + it('should show the creating container', () => { + cluster.updateContainer(null, 'creating'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeFalsy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeTruthy(); + }); + + it('should continue to show `creating` banner with subsequent updates of the same status', () => { + cluster.updateContainer('creating', 'creating'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeFalsy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeTruthy(); + }); + }); + + describe('when cluster is created', () => { + it('should show the success container', () => { + cluster.updateContainer(null, 'created'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeFalsy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeTruthy(); + }); + + it('should not show a banner when status is already `created`', () => { + cluster.updateContainer('created', 'created'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeTruthy(); + }); + }); + + describe('when cluster has error', () => { + it('should show the error container', () => { + cluster.updateContainer(null, 'errored', 'this is an error'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeFalsy(); + + expect( + cluster.errorReasonContainer.textContent, + ).toContain('this is an error'); + }); + + it('should show `error` banner when previously `creating`', () => { + cluster.updateContainer('creating', 'errored'); + + expect( + cluster.creatingContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.successContainer.classList.contains('hidden'), + ).toBeTruthy(); + expect( + cluster.errorContainer.classList.contains('hidden'), + ).toBeFalsy(); + }); + }); + }); + + describe('installApplication', () => { + it('tries to install helm', (done) => { + spyOn(cluster.service, 'installApplication').and.returnValue(Promise.resolve()); + expect(cluster.store.state.applications.helm.requestStatus).toEqual(null); + + cluster.installApplication('helm'); + + expect(cluster.store.state.applications.helm.requestStatus).toEqual(REQUEST_LOADING); + expect(cluster.store.state.applications.helm.requestReason).toEqual(null); + expect(cluster.service.installApplication).toHaveBeenCalledWith('helm'); + + getSetTimeoutPromise() + .then(() => { + expect(cluster.store.state.applications.helm.requestStatus).toEqual(REQUEST_SUCCESS); + expect(cluster.store.state.applications.helm.requestReason).toEqual(null); + }) + .then(done) + .catch(done.fail); + }); + + it('tries to install ingress', (done) => { + spyOn(cluster.service, 'installApplication').and.returnValue(Promise.resolve()); + expect(cluster.store.state.applications.ingress.requestStatus).toEqual(null); + + cluster.installApplication('ingress'); + + expect(cluster.store.state.applications.ingress.requestStatus).toEqual(REQUEST_LOADING); + expect(cluster.store.state.applications.ingress.requestReason).toEqual(null); + expect(cluster.service.installApplication).toHaveBeenCalledWith('ingress'); + + getSetTimeoutPromise() + .then(() => { + expect(cluster.store.state.applications.ingress.requestStatus).toEqual(REQUEST_SUCCESS); + expect(cluster.store.state.applications.ingress.requestReason).toEqual(null); + }) + .then(done) + .catch(done.fail); + }); + + it('tries to install runner', (done) => { + spyOn(cluster.service, 'installApplication').and.returnValue(Promise.resolve()); + expect(cluster.store.state.applications.runner.requestStatus).toEqual(null); + + cluster.installApplication('runner'); + + expect(cluster.store.state.applications.runner.requestStatus).toEqual(REQUEST_LOADING); + expect(cluster.store.state.applications.runner.requestReason).toEqual(null); + expect(cluster.service.installApplication).toHaveBeenCalledWith('runner'); + + getSetTimeoutPromise() + .then(() => { + expect(cluster.store.state.applications.runner.requestStatus).toEqual(REQUEST_SUCCESS); + expect(cluster.store.state.applications.runner.requestReason).toEqual(null); + }) + .then(done) + .catch(done.fail); + }); + + it('sets error request status when the request fails', (done) => { + spyOn(cluster.service, 'installApplication').and.returnValue(Promise.reject(new Error('STUBBED ERROR'))); + expect(cluster.store.state.applications.helm.requestStatus).toEqual(null); + + cluster.installApplication('helm'); + + expect(cluster.store.state.applications.helm.requestStatus).toEqual(REQUEST_LOADING); + expect(cluster.store.state.applications.helm.requestReason).toEqual(null); + expect(cluster.service.installApplication).toHaveBeenCalled(); + + getSetTimeoutPromise() + .then(() => { + expect(cluster.store.state.applications.helm.requestStatus).toEqual(REQUEST_FAILURE); + expect(cluster.store.state.applications.helm.requestReason).toBeDefined(); + }) + .then(done) + .catch(done.fail); + }); + }); +}); diff --git a/spec/javascripts/clusters/components/application_row_spec.js b/spec/javascripts/clusters/components/application_row_spec.js new file mode 100644 index 00000000000..e671c18e1a5 --- /dev/null +++ b/spec/javascripts/clusters/components/application_row_spec.js @@ -0,0 +1,237 @@ +import Vue from 'vue'; +import eventHub from '~/clusters/event_hub'; +import { + APPLICATION_NOT_INSTALLABLE, + APPLICATION_SCHEDULED, + APPLICATION_INSTALLABLE, + APPLICATION_INSTALLING, + APPLICATION_INSTALLED, + APPLICATION_ERROR, + REQUEST_LOADING, + REQUEST_SUCCESS, + REQUEST_FAILURE, +} from '~/clusters/constants'; +import applicationRow from '~/clusters/components/application_row.vue'; +import mountComponent from '../../helpers/vue_mount_component_helper'; +import { DEFAULT_APPLICATION_STATE } from '../services/mock_data'; + +describe('Application Row', () => { + let vm; + let ApplicationRow; + + beforeEach(() => { + ApplicationRow = Vue.extend(applicationRow); + }); + + afterEach(() => { + vm.$destroy(); + }); + + describe('Title', () => { + it('shows title', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + titleLink: null, + }); + const title = vm.$el.querySelector('.js-cluster-application-title'); + + expect(title.tagName).toEqual('SPAN'); + expect(title.textContent.trim()).toEqual(DEFAULT_APPLICATION_STATE.title); + }); + + it('shows title link', () => { + expect(DEFAULT_APPLICATION_STATE.titleLink).toBeDefined(); + + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + }); + const title = vm.$el.querySelector('.js-cluster-application-title'); + + expect(title.tagName).toEqual('A'); + expect(title.textContent.trim()).toEqual(DEFAULT_APPLICATION_STATE.title); + }); + }); + + describe('Install button', () => { + it('has indeterminate state on page load', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: null, + }); + + expect(vm.installButtonLabel).toBeUndefined(); + }); + + it('has disabled "Install" when APPLICATION_NOT_INSTALLABLE', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_NOT_INSTALLABLE, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has enabled "Install" when APPLICATION_INSTALLABLE', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(false); + }); + + it('has loading "Installing" when APPLICATION_SCHEDULED', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_SCHEDULED, + }); + + expect(vm.installButtonLabel).toEqual('Installing'); + expect(vm.installButtonLoading).toEqual(true); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has loading "Installing" when APPLICATION_INSTALLING', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLING, + }); + + expect(vm.installButtonLabel).toEqual('Installing'); + expect(vm.installButtonLoading).toEqual(true); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has disabled "Installed" when APPLICATION_INSTALLED', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLED, + }); + + expect(vm.installButtonLabel).toEqual('Installed'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has enabled "Install" when APPLICATION_ERROR', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_ERROR, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(false); + }); + + it('has loading "Install" when REQUEST_LOADING', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + requestStatus: REQUEST_LOADING, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(true); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has disabled "Install" when REQUEST_SUCCESS', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + requestStatus: REQUEST_SUCCESS, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(true); + }); + + it('has enabled "Install" when REQUEST_FAILURE (so you can try installing again)', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + requestStatus: REQUEST_FAILURE, + }); + + expect(vm.installButtonLabel).toEqual('Install'); + expect(vm.installButtonLoading).toEqual(false); + expect(vm.installButtonDisabled).toEqual(false); + }); + + it('clicking install button emits event', () => { + spyOn(eventHub, '$emit'); + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + }); + const installButton = vm.$el.querySelector('.js-cluster-application-install-button'); + + installButton.click(); + + expect(eventHub.$emit).toHaveBeenCalledWith('installApplication', DEFAULT_APPLICATION_STATE.id); + }); + + it('clicking disabled install button emits nothing', () => { + spyOn(eventHub, '$emit'); + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLING, + }); + const installButton = vm.$el.querySelector('.js-cluster-application-install-button'); + + expect(vm.installButtonDisabled).toEqual(true); + + installButton.click(); + + expect(eventHub.$emit).not.toHaveBeenCalled(); + }); + }); + + describe('Error block', () => { + it('does not show error block when there is no error', () => { + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: null, + requestStatus: null, + }); + const generalErrorMessage = vm.$el.querySelector('.js-cluster-application-general-error-message'); + + expect(generalErrorMessage).toBeNull(); + }); + + it('shows status reason when APPLICATION_ERROR', () => { + const statusReason = 'We broke it 0.0'; + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_ERROR, + statusReason, + }); + const generalErrorMessage = vm.$el.querySelector('.js-cluster-application-general-error-message'); + const statusErrorMessage = vm.$el.querySelector('.js-cluster-application-status-error-message'); + + expect(generalErrorMessage.textContent.trim()).toEqual(`Something went wrong while installing ${DEFAULT_APPLICATION_STATE.title}`); + expect(statusErrorMessage.textContent.trim()).toEqual(statusReason); + }); + + it('shows request reason when REQUEST_FAILURE', () => { + const requestReason = 'We broke thre request 0.0'; + vm = mountComponent(ApplicationRow, { + ...DEFAULT_APPLICATION_STATE, + status: APPLICATION_INSTALLABLE, + requestStatus: REQUEST_FAILURE, + requestReason, + }); + const generalErrorMessage = vm.$el.querySelector('.js-cluster-application-general-error-message'); + const requestErrorMessage = vm.$el.querySelector('.js-cluster-application-request-error-message'); + + expect(generalErrorMessage.textContent.trim()).toEqual(`Something went wrong while installing ${DEFAULT_APPLICATION_STATE.title}`); + expect(requestErrorMessage.textContent.trim()).toEqual(requestReason); + }); + }); +}); diff --git a/spec/javascripts/clusters/components/applications_spec.js b/spec/javascripts/clusters/components/applications_spec.js new file mode 100644 index 00000000000..7460da031c4 --- /dev/null +++ b/spec/javascripts/clusters/components/applications_spec.js @@ -0,0 +1,42 @@ +import Vue from 'vue'; +import applications from '~/clusters/components/applications.vue'; +import mountComponent from '../../helpers/vue_mount_component_helper'; + +describe('Applications', () => { + let vm; + let Applications; + + beforeEach(() => { + Applications = Vue.extend(applications); + }); + + afterEach(() => { + vm.$destroy(); + }); + + describe('', () => { + beforeEach(() => { + vm = mountComponent(Applications, { + applications: { + helm: { title: 'Helm Tiller' }, + ingress: { title: 'Ingress' }, + runner: { title: 'GitLab Runner' }, + }, + }); + }); + + it('renders a row for Helm Tiller', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-helm')).toBeDefined(); + }); + + it('renders a row for Ingress', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-ingress')).toBeDefined(); + }); + + /* * / + it('renders a row for GitLab Runner', () => { + expect(vm.$el.querySelector('.js-cluster-application-row-runner')).toBeDefined(); + }); + /* */ + }); +}); diff --git a/spec/javascripts/clusters/services/mock_data.js b/spec/javascripts/clusters/services/mock_data.js new file mode 100644 index 00000000000..af6b6a73819 --- /dev/null +++ b/spec/javascripts/clusters/services/mock_data.js @@ -0,0 +1,50 @@ +import { + APPLICATION_INSTALLABLE, + APPLICATION_INSTALLING, + APPLICATION_ERROR, +} from '~/clusters/constants'; + +const CLUSTERS_MOCK_DATA = { + GET: { + '/gitlab-org/gitlab-shell/clusters/1/status.json': { + data: { + status: 'errored', + status_reason: 'Failed to request to CloudPlatform.', + applications: [{ + name: 'helm', + status: APPLICATION_INSTALLABLE, + status_reason: null, + }, { + name: 'ingress', + status: APPLICATION_ERROR, + status_reason: 'Cannot connect', + }, { + name: 'runner', + status: APPLICATION_INSTALLING, + status_reason: null, + }], + }, + }, + }, + POST: { + '/gitlab-org/gitlab-shell/clusters/1/applications/helm': { }, + '/gitlab-org/gitlab-shell/clusters/1/applications/ingress': { }, + '/gitlab-org/gitlab-shell/clusters/1/applications/runner': { }, + }, +}; + +const DEFAULT_APPLICATION_STATE = { + id: 'some-app', + title: 'My App', + titleLink: 'https://about.gitlab.com/', + description: 'Some description about this interesting application!', + status: null, + statusReason: null, + requestStatus: null, + requestReason: null, +}; + +export { + CLUSTERS_MOCK_DATA, + DEFAULT_APPLICATION_STATE, +}; diff --git a/spec/javascripts/clusters/stores/clusters_store_spec.js b/spec/javascripts/clusters/stores/clusters_store_spec.js new file mode 100644 index 00000000000..cb8b3d38e2e --- /dev/null +++ b/spec/javascripts/clusters/stores/clusters_store_spec.js @@ -0,0 +1,89 @@ +import ClustersStore from '~/clusters/stores/clusters_store'; +import { APPLICATION_INSTALLING } from '~/clusters/constants'; +import { CLUSTERS_MOCK_DATA } from '../services/mock_data'; + +describe('Clusters Store', () => { + let store; + + beforeEach(() => { + store = new ClustersStore(); + }); + + describe('updateStatus', () => { + it('should store new status', () => { + expect(store.state.status).toEqual(null); + + const newStatus = 'errored'; + store.updateStatus(newStatus); + + expect(store.state.status).toEqual(newStatus); + }); + }); + + describe('updateStatusReason', () => { + it('should store new reason', () => { + expect(store.state.statusReason).toEqual(null); + + const newReason = 'Something went wrong!'; + store.updateStatusReason(newReason); + + expect(store.state.statusReason).toEqual(newReason); + }); + }); + + describe('updateAppProperty', () => { + it('should store new request status', () => { + expect(store.state.applications.helm.requestStatus).toEqual(null); + + const newStatus = APPLICATION_INSTALLING; + store.updateAppProperty('helm', 'requestStatus', newStatus); + + expect(store.state.applications.helm.requestStatus).toEqual(newStatus); + }); + + it('should store new request reason', () => { + expect(store.state.applications.helm.requestReason).toEqual(null); + + const newReason = 'We broke it.'; + store.updateAppProperty('helm', 'requestReason', newReason); + + expect(store.state.applications.helm.requestReason).toEqual(newReason); + }); + }); + + describe('updateStateFromServer', () => { + it('should store new polling data from server', () => { + const mockResponseData = CLUSTERS_MOCK_DATA.GET['/gitlab-org/gitlab-shell/clusters/1/status.json'].data; + store.updateStateFromServer(mockResponseData); + + expect(store.state).toEqual({ + helpPath: null, + status: mockResponseData.status, + statusReason: mockResponseData.status_reason, + applications: { + helm: { + title: 'Helm Tiller', + status: mockResponseData.applications[0].status, + statusReason: mockResponseData.applications[0].status_reason, + requestStatus: null, + requestReason: null, + }, + ingress: { + title: 'Ingress', + status: mockResponseData.applications[1].status, + statusReason: mockResponseData.applications[1].status_reason, + requestStatus: null, + requestReason: null, + }, + runner: { + title: 'GitLab Runner', + status: mockResponseData.applications[2].status, + statusReason: mockResponseData.applications[2].status_reason, + requestStatus: null, + requestReason: null, + }, + }, + }); + }); + }); +}); diff --git a/spec/javascripts/clusters_spec.js b/spec/javascripts/clusters_spec.js deleted file mode 100644 index eb1cd6eb804..00000000000 --- a/spec/javascripts/clusters_spec.js +++ /dev/null @@ -1,79 +0,0 @@ -import Clusters from '~/clusters'; - -describe('Clusters', () => { - let cluster; - preloadFixtures('clusters/show_cluster.html.raw'); - - beforeEach(() => { - loadFixtures('clusters/show_cluster.html.raw'); - cluster = new Clusters(); - }); - - describe('toggle', () => { - it('should update the button and the input field on click', () => { - cluster.toggleButton.click(); - - expect( - cluster.toggleButton.classList, - ).not.toContain('checked'); - - expect( - cluster.toggleInput.getAttribute('value'), - ).toEqual('false'); - }); - }); - - describe('updateContainer', () => { - describe('when creating cluster', () => { - it('should show the creating container', () => { - cluster.updateContainer('creating'); - - expect( - cluster.creatingContainer.classList.contains('hidden'), - ).toBeFalsy(); - expect( - cluster.successContainer.classList.contains('hidden'), - ).toBeTruthy(); - expect( - cluster.errorContainer.classList.contains('hidden'), - ).toBeTruthy(); - }); - }); - - describe('when cluster is created', () => { - it('should show the success container', () => { - cluster.updateContainer('created'); - - expect( - cluster.creatingContainer.classList.contains('hidden'), - ).toBeTruthy(); - expect( - cluster.successContainer.classList.contains('hidden'), - ).toBeFalsy(); - expect( - cluster.errorContainer.classList.contains('hidden'), - ).toBeTruthy(); - }); - }); - - describe('when cluster has error', () => { - it('should show the error container', () => { - cluster.updateContainer('errored', 'this is an error'); - - expect( - cluster.creatingContainer.classList.contains('hidden'), - ).toBeTruthy(); - expect( - cluster.successContainer.classList.contains('hidden'), - ).toBeTruthy(); - expect( - cluster.errorContainer.classList.contains('hidden'), - ).toBeFalsy(); - - expect( - cluster.errorReasonContainer.textContent, - ).toContain('this is an error'); - }); - }); - }); -}); diff --git a/spec/javascripts/smart_interval_spec.js b/spec/javascripts/smart_interval_spec.js index 7833bf3fb04..1c87fcec245 100644 --- a/spec/javascripts/smart_interval_spec.js +++ b/spec/javascripts/smart_interval_spec.js @@ -1,6 +1,6 @@ -import '~/smart_interval'; +import SmartInterval from '~/smart_interval'; -(() => { +describe('SmartInterval', function () { const DEFAULT_MAX_INTERVAL = 100; const DEFAULT_STARTING_INTERVAL = 5; const DEFAULT_SHORT_TIMEOUT = 75; @@ -9,7 +9,7 @@ import '~/smart_interval'; function createDefaultSmartInterval(config) { const defaultParams = { - callback: () => {}, + callback: () => Promise.resolve(), startingInterval: DEFAULT_STARTING_INTERVAL, maxInterval: DEFAULT_MAX_INTERVAL, incrementByFactorOf: DEFAULT_INCREMENT_FACTOR, @@ -22,158 +22,171 @@ import '~/smart_interval'; _.extend(defaultParams, config); } - return new gl.SmartInterval(defaultParams); + return new SmartInterval(defaultParams); } - describe('SmartInterval', function () { - describe('Increment Interval', function () { - beforeEach(function () { - this.smartInterval = createDefaultSmartInterval(); - }); + describe('Increment Interval', function () { + beforeEach(function () { + this.smartInterval = createDefaultSmartInterval(); + }); - it('should increment the interval delay', function (done) { - const interval = this.smartInterval; - setTimeout(() => { - const intervalConfig = this.smartInterval.cfg; - const iterationCount = 4; - const maxIntervalAfterIterations = intervalConfig.startingInterval * - (intervalConfig.incrementByFactorOf ** (iterationCount - 1)); // 40 - const currentInterval = interval.getCurrentInterval(); - - // Provide some flexibility for performance of testing environment - expect(currentInterval).toBeGreaterThan(intervalConfig.startingInterval); - expect(currentInterval <= maxIntervalAfterIterations).toBeTruthy(); - - done(); - }, DEFAULT_SHORT_TIMEOUT); // 4 iterations, increment by 2x = (5 + 10 + 20 + 40) - }); + it('should increment the interval delay', function (done) { + const interval = this.smartInterval; + setTimeout(() => { + const intervalConfig = this.smartInterval.cfg; + const iterationCount = 4; + const maxIntervalAfterIterations = intervalConfig.startingInterval * + (intervalConfig.incrementByFactorOf ** (iterationCount - 1)); // 40 + const currentInterval = interval.getCurrentInterval(); + + // Provide some flexibility for performance of testing environment + expect(currentInterval).toBeGreaterThan(intervalConfig.startingInterval); + expect(currentInterval <= maxIntervalAfterIterations).toBeTruthy(); + + done(); + }, DEFAULT_SHORT_TIMEOUT); // 4 iterations, increment by 2x = (5 + 10 + 20 + 40) + }); - it('should not increment past maxInterval', function (done) { - const interval = this.smartInterval; + it('should not increment past maxInterval', function (done) { + const interval = this.smartInterval; - setTimeout(() => { - const currentInterval = interval.getCurrentInterval(); - expect(currentInterval).toBe(interval.cfg.maxInterval); + setTimeout(() => { + const currentInterval = interval.getCurrentInterval(); + expect(currentInterval).toBe(interval.cfg.maxInterval); - done(); - }, DEFAULT_LONG_TIMEOUT); - }); + done(); + }, DEFAULT_LONG_TIMEOUT); }); - describe('Public methods', function () { - beforeEach(function () { - this.smartInterval = createDefaultSmartInterval(); + it('does not increment while waiting for callback', function () { + jasmine.clock().install(); + + const smartInterval = createDefaultSmartInterval({ + callback: () => new Promise($.noop), }); - it('should cancel an interval', function (done) { - const interval = this.smartInterval; + jasmine.clock().tick(DEFAULT_SHORT_TIMEOUT); + + const oneInterval = smartInterval.cfg.startingInterval * DEFAULT_INCREMENT_FACTOR; + expect(smartInterval.getCurrentInterval()).toEqual(oneInterval); - setTimeout(() => { - interval.cancel(); + jasmine.clock().uninstall(); + }); + }); - const intervalId = interval.state.intervalId; - const currentInterval = interval.getCurrentInterval(); - const intervalLowerLimit = interval.cfg.startingInterval; + describe('Public methods', function () { + beforeEach(function () { + this.smartInterval = createDefaultSmartInterval(); + }); - expect(intervalId).toBeUndefined(); - expect(currentInterval).toBe(intervalLowerLimit); + it('should cancel an interval', function (done) { + const interval = this.smartInterval; - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + setTimeout(() => { + interval.cancel(); - it('should resume an interval', function (done) { - const interval = this.smartInterval; + const intervalId = interval.state.intervalId; + const currentInterval = interval.getCurrentInterval(); + const intervalLowerLimit = interval.cfg.startingInterval; - setTimeout(() => { - interval.cancel(); + expect(intervalId).toBeUndefined(); + expect(currentInterval).toBe(intervalLowerLimit); - interval.resume(); + done(); + }, DEFAULT_SHORT_TIMEOUT); + }); - const intervalId = interval.state.intervalId; + it('should resume an interval', function (done) { + const interval = this.smartInterval; - expect(intervalId).toBeTruthy(); + setTimeout(() => { + interval.cancel(); - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + interval.resume(); + + const intervalId = interval.state.intervalId; + + expect(intervalId).toBeTruthy(); + + done(); + }, DEFAULT_SHORT_TIMEOUT); }); + }); - describe('DOM Events', function () { - beforeEach(function () { - // This ensures DOM and DOM events are initialized for these specs. - setFixtures('<div></div>'); + describe('DOM Events', function () { + beforeEach(function () { + // This ensures DOM and DOM events are initialized for these specs. + setFixtures('<div></div>'); - this.smartInterval = createDefaultSmartInterval(); - }); + this.smartInterval = createDefaultSmartInterval(); + }); - it('should pause when page is not visible', function (done) { - const interval = this.smartInterval; + it('should pause when page is not visible', function (done) { + const interval = this.smartInterval; - setTimeout(() => { - expect(interval.state.intervalId).toBeTruthy(); + setTimeout(() => { + expect(interval.state.intervalId).toBeTruthy(); - // simulates triggering of visibilitychange event - interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); + // simulates triggering of visibilitychange event + interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); - expect(interval.state.intervalId).toBeUndefined(); - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + expect(interval.state.intervalId).toBeUndefined(); + done(); + }, DEFAULT_SHORT_TIMEOUT); + }); - it('should change to the hidden interval when page is not visible', function (done) { - const HIDDEN_INTERVAL = 1500; - const interval = createDefaultSmartInterval({ hiddenInterval: HIDDEN_INTERVAL }); + it('should change to the hidden interval when page is not visible', function (done) { + const HIDDEN_INTERVAL = 1500; + const interval = createDefaultSmartInterval({ hiddenInterval: HIDDEN_INTERVAL }); - setTimeout(() => { - expect(interval.state.intervalId).toBeTruthy(); - expect(interval.getCurrentInterval() >= DEFAULT_STARTING_INTERVAL && - interval.getCurrentInterval() <= DEFAULT_MAX_INTERVAL).toBeTruthy(); + setTimeout(() => { + expect(interval.state.intervalId).toBeTruthy(); + expect(interval.getCurrentInterval() >= DEFAULT_STARTING_INTERVAL && + interval.getCurrentInterval() <= DEFAULT_MAX_INTERVAL).toBeTruthy(); - // simulates triggering of visibilitychange event - interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); + // simulates triggering of visibilitychange event + interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); - expect(interval.state.intervalId).toBeTruthy(); - expect(interval.getCurrentInterval()).toBe(HIDDEN_INTERVAL); - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + expect(interval.state.intervalId).toBeTruthy(); + expect(interval.getCurrentInterval()).toBe(HIDDEN_INTERVAL); + done(); + }, DEFAULT_SHORT_TIMEOUT); + }); - it('should resume when page is becomes visible at the previous interval', function (done) { - const interval = this.smartInterval; + it('should resume when page is becomes visible at the previous interval', function (done) { + const interval = this.smartInterval; - setTimeout(() => { - expect(interval.state.intervalId).toBeTruthy(); + setTimeout(() => { + expect(interval.state.intervalId).toBeTruthy(); - // simulates triggering of visibilitychange event - interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); + // simulates triggering of visibilitychange event + interval.handleVisibilityChange({ target: { visibilityState: 'hidden' } }); - expect(interval.state.intervalId).toBeUndefined(); + expect(interval.state.intervalId).toBeUndefined(); - // simulates triggering of visibilitychange event - interval.handleVisibilityChange({ target: { visibilityState: 'visible' } }); + // simulates triggering of visibilitychange event + interval.handleVisibilityChange({ target: { visibilityState: 'visible' } }); - expect(interval.state.intervalId).toBeTruthy(); + expect(interval.state.intervalId).toBeTruthy(); - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + done(); + }, DEFAULT_SHORT_TIMEOUT); + }); - it('should cancel on page unload', function (done) { - const interval = this.smartInterval; + it('should cancel on page unload', function (done) { + const interval = this.smartInterval; - setTimeout(() => { - $(document).triggerHandler('beforeunload'); - expect(interval.state.intervalId).toBeUndefined(); - expect(interval.getCurrentInterval()).toBe(interval.cfg.startingInterval); - done(); - }, DEFAULT_SHORT_TIMEOUT); - }); + setTimeout(() => { + $(document).triggerHandler('beforeunload'); + expect(interval.state.intervalId).toBeUndefined(); + expect(interval.getCurrentInterval()).toBe(interval.cfg.startingInterval); + done(); + }, DEFAULT_SHORT_TIMEOUT); + }); - it('should execute callback before first interval', function () { - const interval = createDefaultSmartInterval({ immediateExecution: true }); - expect(interval.cfg.immediateExecution).toBeFalsy(); - }); + it('should execute callback before first interval', function () { + const interval = createDefaultSmartInterval({ immediateExecution: true }); + expect(interval.cfg.immediateExecution).toBeFalsy(); }); }); -})(window.gl || (window.gl = {})); +}); diff --git a/spec/javascripts/vue_mr_widget/components/mr_widget_pipeline_spec.js b/spec/javascripts/vue_mr_widget/components/mr_widget_pipeline_spec.js index 33ed0cb4342..d7af956c9c1 100644 --- a/spec/javascripts/vue_mr_widget/components/mr_widget_pipeline_spec.js +++ b/spec/javascripts/vue_mr_widget/components/mr_widget_pipeline_spec.js @@ -1,140 +1,147 @@ import Vue from 'vue'; -import pipelineComponent from '~/vue_merge_request_widget/components/mr_widget_pipeline'; +import pipelineComponent from '~/vue_merge_request_widget/components/mr_widget_pipeline.vue'; +import mountComponent from '../../helpers/vue_mount_component_helper'; import mockData from '../mock_data'; -const createComponent = (mr) => { - const Component = Vue.extend(pipelineComponent); - return new Component({ - el: document.createElement('div'), - propsData: { mr }, - }); -}; - describe('MRWidgetPipeline', () => { - describe('props', () => { - it('should have props', () => { - const { mr } = pipelineComponent.props; + let vm; + let Component; - expect(mr.type instanceof Object).toBeTruthy(); - expect(mr.required).toBeTruthy(); - }); + beforeEach(() => { + Component = Vue.extend(pipelineComponent); }); - describe('components', () => { - it('should have components added', () => { - expect(pipelineComponent.components['pipeline-stage']).toBeDefined(); - expect(pipelineComponent.components.ciIcon).toBeDefined(); - }); + afterEach(() => { + vm.$destroy(); }); describe('computed', () => { describe('hasPipeline', () => { it('should return true when there is a pipeline', () => { - expect(Object.keys(mockData.pipeline).length).toBeGreaterThan(0); - - const vm = createComponent({ + vm = mountComponent(Component, { pipeline: mockData.pipeline, + ciStatus: 'success', + hasCi: true, }); - expect(vm.hasPipeline).toBeTruthy(); + expect(vm.hasPipeline).toEqual(true); }); it('should return false when there is no pipeline', () => { - const vm = createComponent({ - pipeline: null, + vm = mountComponent(Component, { + pipeline: {}, }); - expect(vm.hasPipeline).toBeFalsy(); + expect(vm.hasPipeline).toEqual(false); }); }); describe('hasCIError', () => { it('should return false when there is no CI error', () => { - const vm = createComponent({ + vm = mountComponent(Component, { pipeline: mockData.pipeline, - hasCI: true, + hasCi: true, ciStatus: 'success', }); - expect(vm.hasCIError).toBeFalsy(); + expect(vm.hasCIError).toEqual(false); }); it('should return true when there is a CI error', () => { - const vm = createComponent({ + vm = mountComponent(Component, { pipeline: mockData.pipeline, - hasCI: true, + hasCi: true, ciStatus: null, }); - expect(vm.hasCIError).toBeTruthy(); + expect(vm.hasCIError).toEqual(true); }); }); }); - describe('template', () => { - let vm; - let el; - const { pipeline } = mockData; - const mr = { - hasCI: true, - ciStatus: 'success', - pipelineDetailedStatus: pipeline.details.status, - pipeline, - }; - - beforeEach(() => { - vm = createComponent(mr); - el = vm.$el; - }); + describe('rendered output', () => { + it('should render CI error', () => { + vm = mountComponent(Component, { + pipeline: mockData.pipeline, + hasCi: true, + ciStatus: null, + }); - it('should render template elements correctly', () => { - expect(el.classList.contains('mr-widget-heading')).toBeTruthy(); - expect(el.querySelectorAll('.ci-status-icon.ci-status-icon-success').length).toEqual(1); - expect(el.querySelector('.pipeline-id').textContent).toContain(`#${pipeline.id}`); - expect(el.innerText).toContain('passed'); - expect(el.querySelector('.pipeline-id').getAttribute('href')).toEqual(pipeline.path); - expect(el.querySelectorAll('.stage-container').length).toEqual(2); - expect(el.querySelector('.js-ci-error')).toEqual(null); - expect(el.querySelector('.js-commit-link').getAttribute('href')).toEqual(pipeline.commit.commit_path); - expect(el.querySelector('.js-commit-link').textContent).toContain(pipeline.commit.short_id); - expect(el.querySelector('.js-mr-coverage').textContent).toContain(`Coverage ${pipeline.coverage}%`); + expect( + vm.$el.querySelector('.media-body').textContent.trim(), + ).toEqual('Could not connect to the CI server. Please check your settings and try again'); }); - it('should list single stage', (done) => { - pipeline.details.stages.splice(0, 1); + describe('with a pipeline', () => { + beforeEach(() => { + vm = mountComponent(Component, { + pipeline: mockData.pipeline, + hasCi: true, + ciStatus: 'success', + }); + }); + + it('should render pipeline ID', () => { + expect( + vm.$el.querySelector('.pipeline-id').textContent.trim(), + ).toEqual(`#${mockData.pipeline.id}`); + }); + + it('should render pipeline status and commit id', () => { + expect( + vm.$el.querySelector('.media-body').textContent.trim(), + ).toContain(mockData.pipeline.details.status.label); - Vue.nextTick(() => { - expect(el.querySelectorAll('.stage-container button').length).toEqual(1); - done(); + expect( + vm.$el.querySelector('.js-commit-link').textContent.trim(), + ).toEqual(mockData.pipeline.commit.short_id); + + expect( + vm.$el.querySelector('.js-commit-link').getAttribute('href'), + ).toEqual(mockData.pipeline.commit.commit_path); }); - }); - it('should not have stages when there is no stage', (done) => { - vm.mr.pipeline.details.stages = []; + it('should render pipeline graph', () => { + expect(vm.$el.querySelector('.mr-widget-pipeline-graph')).toBeDefined(); + expect(vm.$el.querySelectorAll('.stage-container').length).toEqual(mockData.pipeline.details.stages.length); + }); - Vue.nextTick(() => { - expect(el.querySelectorAll('.stage-container button').length).toEqual(0); - done(); + it('should render coverage information', () => { + expect( + vm.$el.querySelector('.media-body').textContent, + ).toContain(`Coverage ${mockData.pipeline.coverage}`); }); }); - it('should not have coverage text when pipeline has no coverage info', (done) => { - vm.mr.pipeline.coverage = null; + describe('without coverage', () => { + it('should not render a coverage', () => { + const mockCopy = Object.assign({}, mockData); + delete mockCopy.pipeline.coverage; - Vue.nextTick(() => { - expect(el.querySelector('.js-mr-coverage')).toEqual(null); - done(); + vm = mountComponent(Component, { + pipeline: mockCopy.pipeline, + hasCi: true, + ciStatus: 'success', + }); + + expect( + vm.$el.querySelector('.media-body').textContent, + ).not.toContain('Coverage'); }); }); - it('should show CI error when there is a CI error', (done) => { - vm.mr.ciStatus = null; + describe('without a pipeline graph', () => { + it('should not render a pipeline graph', () => { + const mockCopy = Object.assign({}, mockData); + delete mockCopy.pipeline.details.stages; + + vm = mountComponent(Component, { + pipeline: mockCopy.pipeline, + hasCi: true, + ciStatus: 'success', + }); - Vue.nextTick(() => { - expect(el.querySelectorAll('.js-ci-error').length).toEqual(1); - expect(el.innerText).toContain('Could not connect to the CI server'); - expect(el.querySelector('.ci-status-icon svg use').getAttribute('xlink:href')).toContain('status_failed'); - done(); + expect(vm.$el.querySelector('.js-mini-pipeline-graph')).toEqual(null); }); }); }); diff --git a/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js b/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js index e4324e91502..8832dd161c7 100644 --- a/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js +++ b/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js @@ -1,5 +1,4 @@ import Vue from 'vue'; -import MRWidgetService from '~/vue_merge_request_widget/services/mr_widget_service'; import mrWidgetOptions from '~/vue_merge_request_widget/mr_widget_options'; import eventHub from '~/vue_merge_request_widget/event_hub'; import notify from '~/lib/utils/notify'; @@ -121,24 +120,28 @@ describe('mrWidgetOptions', () => { describe('initPolling', () => { it('should call SmartInterval', () => { - spyOn(gl, 'SmartInterval').and.returnValue({ - resume() {}, - stopTimer() {}, - }); + spyOn(vm, 'checkStatus').and.returnValue(Promise.resolve()); + jasmine.clock().install(); vm.initPolling(); + expect(vm.checkStatus).not.toHaveBeenCalled(); + + jasmine.clock().tick(10000); + expect(vm.pollingInterval).toBeDefined(); - expect(gl.SmartInterval).toHaveBeenCalled(); + expect(vm.checkStatus).toHaveBeenCalled(); + + jasmine.clock().uninstall(); }); }); describe('initDeploymentsPolling', () => { it('should call SmartInterval', () => { - spyOn(gl, 'SmartInterval'); + spyOn(vm, 'fetchDeployments').and.returnValue(Promise.resolve()); vm.initDeploymentsPolling(); expect(vm.deploymentsInterval).toBeDefined(); - expect(gl.SmartInterval).toHaveBeenCalled(); + expect(vm.fetchDeployments).toHaveBeenCalled(); }); }); @@ -312,28 +315,6 @@ describe('mrWidgetOptions', () => { expect(vm.pollingInterval.stopTimer).toHaveBeenCalled(); }); }); - - describe('createService', () => { - it('should instantiate a Service', () => { - const endpoints = { - mergePath: '/nice/path', - mergeCheckPath: '/nice/path', - cancelAutoMergePath: '/nice/path', - removeWIPPath: '/nice/path', - sourceBranchPath: '/nice/path', - ciEnvironmentsStatusPath: '/nice/path', - statusPath: '/nice/path', - mergeActionsContentPath: '/nice/path', - }; - - const serviceInstance = vm.createService(endpoints); - const isInstanceOfMRService = serviceInstance instanceof MRWidgetService; - expect(isInstanceOfMRService).toBe(true); - Object.keys(serviceInstance).forEach((key) => { - expect(serviceInstance[key]).toBeDefined(); - }); - }); - }); }); describe('components', () => { diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index 1076c63b5f2..10020511bf8 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -13,6 +13,47 @@ describe Feature do end end + describe '.persisted_names' do + it 'returns the names of the persisted features' do + Feature::FlipperFeature.create!(key: 'foo') + + expect(described_class.persisted_names).to eq(%w[foo]) + end + + it 'returns an empty Array when no features are presisted' do + expect(described_class.persisted_names).to be_empty + end + + it 'caches the feature names when request store is active', :request_store do + Feature::FlipperFeature.create!(key: 'foo') + + expect(Feature::FlipperFeature) + .to receive(:feature_names) + .once + .and_call_original + + 2.times do + expect(described_class.persisted_names).to eq(%w[foo]) + end + end + end + + describe '.persisted?' do + it 'returns true for a persisted feature' do + Feature::FlipperFeature.create!(key: 'foo') + + feature = double(:feature, name: 'foo') + + expect(described_class.persisted?(feature)).to eq(true) + end + + it 'returns false for a feature that is not persisted' do + feature = double(:feature, name: 'foo') + + expect(described_class.persisted?(feature)).to eq(false) + end + end + describe '.all' do let(:features) { Set.new } diff --git a/spec/lib/github/client_spec.rb b/spec/lib/github/client_spec.rb deleted file mode 100644 index b846096fe25..00000000000 --- a/spec/lib/github/client_spec.rb +++ /dev/null @@ -1,34 +0,0 @@ -require 'spec_helper' - -describe Github::Client do - let(:connection) { spy } - let(:rate_limit) { double(get: [false, 1]) } - let(:client) { described_class.new({}) } - let(:results) { double } - let(:response) { double } - - before do - allow(Faraday).to receive(:new).and_return(connection) - allow(Github::RateLimit).to receive(:new).with(connection).and_return(rate_limit) - end - - describe '#get' do - before do - allow(Github::Response).to receive(:new).with(results).and_return(response) - end - - it 'uses a default per_page param' do - expect(connection).to receive(:get).with('/foo', per_page: 100).and_return(results) - - expect(client.get('/foo')).to eq(response) - end - - context 'with per_page given' do - it 'overwrites the default per_page' do - expect(connection).to receive(:get).with('/foo', per_page: 30).and_return(results) - - expect(client.get('/foo', per_page: 30)).to eq(response) - end - end - end -end diff --git a/spec/lib/github/import/legacy_diff_note_spec.rb b/spec/lib/github/import/legacy_diff_note_spec.rb deleted file mode 100644 index 8c50b46cacb..00000000000 --- a/spec/lib/github/import/legacy_diff_note_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'spec_helper' - -describe Github::Import::LegacyDiffNote do - describe '#type' do - it 'returns the original note type' do - expect(described_class.new.type).to eq('LegacyDiffNote') - end - end -end diff --git a/spec/lib/github/import/note_spec.rb b/spec/lib/github/import/note_spec.rb deleted file mode 100644 index fcdccd9e097..00000000000 --- a/spec/lib/github/import/note_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -require 'spec_helper' - -describe Github::Import::Note do - describe '#type' do - it 'returns the original note type' do - expect(described_class.new.type).to eq('Note') - end - end -end diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb new file mode 100644 index 00000000000..7f3bf5fc41c --- /dev/null +++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb @@ -0,0 +1,168 @@ +require 'spec_helper' + +describe Gitlab::BareRepositoryImport::Importer, repository: true do + let!(:admin) { create(:admin) } + let!(:base_dir) { Dir.mktmpdir + '/' } + let(:bare_repository) { Gitlab::BareRepositoryImport::Repository.new(base_dir, File.join(base_dir, "#{project_path}.git")) } + + subject(:importer) { described_class.new(admin, bare_repository) } + + before do + allow(described_class).to receive(:log) + end + + after do + FileUtils.rm_rf(base_dir) + end + + shared_examples 'importing a repository' do + describe '.execute' do + it 'creates a project for a repository in storage' do + FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + fake_importer = double + + expect(described_class).to receive(:new).and_return(fake_importer) + expect(fake_importer).to receive(:create_project_if_needed) + + described_class.execute(base_dir) + end + + it 'skips wiki repos' do + repo_dir = File.join(base_dir, 'the-group', 'the-project.wiki.git') + FileUtils.mkdir_p(File.join(repo_dir)) + + expect(described_class).to receive(:log).with(" * Skipping repo #{repo_dir}") + expect(described_class).not_to receive(:new) + + described_class.execute(base_dir) + end + + context 'without admin users' do + let(:admin) { nil } + + it 'raises an error' do + expect { described_class.execute(base_dir) }.to raise_error(Gitlab::BareRepositoryImport::Importer::NoAdminError) + end + end + end + + describe '#create_project_if_needed' do + it 'starts an import for a project that did not exist' do + expect(importer).to receive(:create_project) + + importer.create_project_if_needed + end + + it 'skips importing when the project already exists' do + project = create(:project, path: 'a-project', namespace: existing_group) + + expect(importer).not_to receive(:create_project) + expect(importer).to receive(:log).with(" * #{project.name} (#{project_path}) exists") + + importer.create_project_if_needed + end + + it 'creates a project with the correct path in the database' do + importer.create_project_if_needed + + expect(Project.find_by_full_path(project_path)).not_to be_nil + end + + it 'creates the Git repo in disk' do + FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + + importer.create_project_if_needed + + project = Project.find_by_full_path(project_path) + + expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.git')) + end + + context 'hashed storage enabled' do + it 'creates a project with the correct path in the database' do + stub_application_setting(hashed_storage_enabled: true) + + importer.create_project_if_needed + + expect(Project.find_by_full_path(project_path)).not_to be_nil + end + end + end + end + + context 'with subgroups', :nested_groups do + let(:project_path) { 'a-group/a-sub-group/a-project' } + + let(:existing_group) do + group = create(:group, path: 'a-group') + create(:group, path: 'a-sub-group', parent: group) + end + + it_behaves_like 'importing a repository' + end + + context 'without subgroups' do + let(:project_path) { 'a-group/a-project' } + let(:existing_group) { create(:group, path: 'a-group') } + + it_behaves_like 'importing a repository' + end + + context 'without groups' do + let(:project_path) { 'a-project' } + + it 'starts an import for a project that did not exist' do + expect(importer).to receive(:create_project) + + importer.create_project_if_needed + end + + it 'creates a project with the correct path in the database' do + importer.create_project_if_needed + + expect(Project.find_by_full_path("#{admin.full_path}/#{project_path}")).not_to be_nil + end + + it 'creates the Git repo in disk' do + FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + + importer.create_project_if_needed + + project = Project.find_by_full_path("#{admin.full_path}/#{project_path}") + + expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.git')) + end + end + + context 'with Wiki' do + let(:project_path) { 'a-group/a-project' } + let(:existing_group) { create(:group, path: 'a-group') } + + it_behaves_like 'importing a repository' + + it 'creates the Wiki git repo in disk' do + FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.git")) + FileUtils.mkdir_p(File.join(base_dir, "#{project_path}.wiki.git")) + + importer.create_project_if_needed + + project = Project.find_by_full_path(project_path) + + expect(File).to exist(File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + end + end + + context 'when subgroups are not available' do + let(:project_path) { 'a-group/a-sub-group/a-project' } + + before do + expect(Group).to receive(:supports_nested_groups?) { false } + end + + describe '#create_project_if_needed' do + it 'raises an error' do + expect { importer.create_project_if_needed }.to raise_error('Nested groups are not supported on MySQL') + end + end + end +end diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb new file mode 100644 index 00000000000..2db737f5fb6 --- /dev/null +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' + +describe ::Gitlab::BareRepositoryImport::Repository do + let(:project_repo_path) { described_class.new('/full/path/', '/full/path/to/repo.git') } + + it 'stores the repo path' do + expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git') + end + + it 'stores the group path' do + expect(project_repo_path.group_path).to eq('to') + end + + it 'stores the project name' do + expect(project_repo_path.project_name).to eq('repo') + end + + it 'stores the wiki path' do + expect(project_repo_path.wiki_path).to eq('/full/path/to/repo.wiki.git') + end + + describe '#wiki?' do + it 'returns true if it is a wiki' do + wiki_path = described_class.new('/full/path/', '/full/path/to/a/b/my.wiki.git') + + expect(wiki_path.wiki?).to eq(true) + end + + it 'returns false if it is not a wiki' do + expect(project_repo_path.wiki?).to eq(false) + end + end + + describe '#hashed?' do + it 'returns true if it is a hashed folder' do + path = described_class.new('/full/path/', '/full/path/@hashed/my.repo.git') + + expect(path.hashed?).to eq(true) + end + + it 'returns false if it is not a hashed folder' do + expect(project_repo_path.hashed?).to eq(false) + end + end + + describe '#project_full_path' do + it 'returns the project full path' do + expect(project_repo_path.repo_path).to eq('/full/path/to/repo.git') + end + end +end diff --git a/spec/lib/gitlab/bare_repository_importer_spec.rb b/spec/lib/gitlab/bare_repository_importer_spec.rb deleted file mode 100644 index 36d1844b5b1..00000000000 --- a/spec/lib/gitlab/bare_repository_importer_spec.rb +++ /dev/null @@ -1,100 +0,0 @@ -require 'spec_helper' - -describe Gitlab::BareRepositoryImporter, repository: true do - subject(:importer) { described_class.new('default', project_path) } - - let!(:admin) { create(:admin) } - - before do - allow(described_class).to receive(:log) - end - - shared_examples 'importing a repository' do - describe '.execute' do - it 'creates a project for a repository in storage' do - FileUtils.mkdir_p(File.join(TestEnv.repos_path, "#{project_path}.git")) - fake_importer = double - - expect(described_class).to receive(:new).with('default', project_path) - .and_return(fake_importer) - expect(fake_importer).to receive(:create_project_if_needed) - - described_class.execute - end - - it 'skips wiki repos' do - FileUtils.mkdir_p(File.join(TestEnv.repos_path, 'the-group', 'the-project.wiki.git')) - - expect(described_class).to receive(:log).with(' * Skipping wiki repo') - expect(described_class).not_to receive(:new) - - described_class.execute - end - end - - describe '#initialize' do - context 'without admin users' do - let(:admin) { nil } - - it 'raises an error' do - expect { importer }.to raise_error(Gitlab::BareRepositoryImporter::NoAdminError) - end - end - end - - describe '#create_project_if_needed' do - it 'starts an import for a project that did not exist' do - expect(importer).to receive(:create_project) - - importer.create_project_if_needed - end - - it 'skips importing when the project already exists' do - project = create(:project, path: 'a-project', namespace: existing_group) - - expect(importer).not_to receive(:create_project) - expect(importer).to receive(:log).with(" * #{project.name} (#{project_path}) exists") - - importer.create_project_if_needed - end - - it 'creates a project with the correct path in the database' do - importer.create_project_if_needed - - expect(Project.find_by_full_path(project_path)).not_to be_nil - end - end - end - - context 'with subgroups', :nested_groups do - let(:project_path) { 'a-group/a-sub-group/a-project' } - - let(:existing_group) do - group = create(:group, path: 'a-group') - create(:group, path: 'a-sub-group', parent: group) - end - - it_behaves_like 'importing a repository' - end - - context 'without subgroups' do - let(:project_path) { 'a-group/a-project' } - let(:existing_group) { create(:group, path: 'a-group') } - - it_behaves_like 'importing a repository' - end - - context 'when subgroups are not available' do - let(:project_path) { 'a-group/a-sub-group/a-project' } - - before do - expect(Group).to receive(:supports_nested_groups?) { false } - end - - describe '#create_project_if_needed' do - it 'raises an error' do - expect { importer.create_project_if_needed }.to raise_error('Nested groups are not supported on MySQL') - end - end - end -end diff --git a/spec/lib/gitlab/ci/status/build/factory_spec.rb b/spec/lib/gitlab/ci/status/build/factory_spec.rb index 2b32e47e9ba..d196bc6a4c2 100644 --- a/spec/lib/gitlab/ci/status/build/factory_spec.rb +++ b/spec/lib/gitlab/ci/status/build/factory_spec.rb @@ -84,7 +84,7 @@ describe Gitlab::Ci::Status::Build::Factory do it 'fabricates status with correct details' do expect(status.text).to eq 'failed' - expect(status.icon).to eq 'warning' + expect(status.icon).to eq 'status_warning' expect(status.favicon).to eq 'favicon_status_failed' expect(status.label).to eq 'failed (allowed to fail)' expect(status).to have_details diff --git a/spec/lib/gitlab/ci/status/build/failed_allowed_spec.rb b/spec/lib/gitlab/ci/status/build/failed_allowed_spec.rb index 79a65fc67e8..99a5a7e4aca 100644 --- a/spec/lib/gitlab/ci/status/build/failed_allowed_spec.rb +++ b/spec/lib/gitlab/ci/status/build/failed_allowed_spec.rb @@ -18,7 +18,7 @@ describe Gitlab::Ci::Status::Build::FailedAllowed do describe '#icon' do it 'returns a warning icon' do - expect(subject.icon).to eq 'warning' + expect(subject.icon).to eq 'status_warning' end end diff --git a/spec/lib/gitlab/database/grant_spec.rb b/spec/lib/gitlab/database/grant_spec.rb index 651da3e8476..5ebf3f399b6 100644 --- a/spec/lib/gitlab/database/grant_spec.rb +++ b/spec/lib/gitlab/database/grant_spec.rb @@ -1,16 +1,6 @@ require 'spec_helper' describe Gitlab::Database::Grant do - describe '.scope_to_current_user' do - it 'scopes the relation to the current user' do - user = Gitlab::Database.username - column = Gitlab::Database.postgresql? ? :grantee : :User - names = described_class.scope_to_current_user.pluck(column).uniq - - expect(names).to eq([user]) - end - end - describe '.create_and_execute_trigger' do it 'returns true when the user can create and execute a trigger' do # We assume the DB/user is set up correctly so that triggers can be @@ -18,13 +8,11 @@ describe Gitlab::Database::Grant do expect(described_class.create_and_execute_trigger?('users')).to eq(true) end - it 'returns false when the user can not create and/or execute a trigger' do - allow(described_class).to receive(:scope_to_current_user) - .and_return(described_class.none) - - result = described_class.create_and_execute_trigger?('kittens') - - expect(result).to eq(false) + it 'returns false when the user can not create and/or execute a trigger', :postgresql do + # In case of MySQL the user may have SUPER permissions, making it + # impossible to have `false` returned when running tests; hence we only + # run these tests on PostgreSQL. + expect(described_class.create_and_execute_trigger?('foo')).to eq(false) end end end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 7aeb85b8f5a..fcddfad3f9f 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -202,6 +202,26 @@ describe Gitlab::Database do it 'handles non-UTF-8 data' do expect { described_class.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error end + + context 'when using PostgreSQL' do + before do + allow(described_class).to receive(:mysql?).and_return(false) + end + + it 'allows the returning of the IDs of the inserted rows' do + result = double(:result, values: [['10']]) + + expect(connection) + .to receive(:execute) + .with(/RETURNING id/) + .and_return(result) + + ids = described_class + .bulk_insert('test', [{ number: 10 }], return_ids: true) + + expect(ids).to eq([10]) + end + end end describe '.create_connection_pool' do diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 96e162ac087..ee14b528ec2 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -648,6 +648,21 @@ describe Gitlab::Git::Repository, seed_helper: true do end end + describe '#remote_exists?' do + before(:all) do + @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') + @repo.add_remote("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL) + end + + it 'returns true for an existing remote' do + expect(@repo.remote_exists?('new_remote')).to eq(true) + end + + it 'returns false for a non-existing remote' do + expect(@repo.remote_exists?('foo')).to eq(false) + end + end + describe "#log" do let(:commit_with_old_name) do Gitlab::Git::Commit.decorate(repository, @commit_with_old_name_id) diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb new file mode 100644 index 00000000000..91229d9c7d4 --- /dev/null +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::BulkImporting do + let(:importer) do + Class.new { include(Gitlab::GithubImport::BulkImporting) }.new + end + + describe '#build_database_rows' do + it 'returns an Array containing the rows to insert' do + object = double(:object, title: 'Foo') + + expect(importer) + .to receive(:build) + .with(object) + .and_return({ title: 'Foo' }) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) + + enum = [[object, 1]].to_enum + + expect(importer.build_database_rows(enum)).to eq([{ title: 'Foo' }]) + end + + it 'does not import objects that have already been imported' do + object = double(:object, title: 'Foo') + + expect(importer) + .not_to receive(:build) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(true) + + enum = [[object, 1]].to_enum + + expect(importer.build_database_rows(enum)).to be_empty + end + end + + describe '#bulk_insert' do + it 'bulk inserts rows into the database' do + rows = [{ title: 'Foo' }] * 10 + model = double(:model, table_name: 'kittens') + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .ordered + .with('kittens', rows.first(5)) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .ordered + .with('kittens', rows.last(5)) + + importer.bulk_insert(model, rows, batch_size: 5) + end + end +end diff --git a/spec/lib/gitlab/github_import/caching_spec.rb b/spec/lib/gitlab/github_import/caching_spec.rb new file mode 100644 index 00000000000..70ecdc16da1 --- /dev/null +++ b/spec/lib/gitlab/github_import/caching_spec.rb @@ -0,0 +1,117 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Caching, :clean_gitlab_redis_cache do + describe '.read' do + it 'reads a value from the cache' do + described_class.write('foo', 'bar') + + expect(described_class.read('foo')).to eq('bar') + end + + it 'returns nil if the cache key does not exist' do + expect(described_class.read('foo')).to be_nil + end + + it 'refreshes the cache key if a value is present' do + described_class.write('foo', 'bar') + + redis = double(:redis) + + expect(redis).to receive(:get).with(/foo/).and_return('bar') + expect(redis).to receive(:expire).with(/foo/, described_class::TIMEOUT) + expect(Gitlab::Redis::Cache).to receive(:with).twice.and_yield(redis) + + described_class.read('foo') + end + + it 'does not refresh the cache key if a value is empty' do + described_class.write('foo', nil) + + redis = double(:redis) + + expect(redis).to receive(:get).with(/foo/).and_return('') + expect(redis).not_to receive(:expire) + expect(Gitlab::Redis::Cache).to receive(:with).and_yield(redis) + + described_class.read('foo') + end + end + + describe '.read_integer' do + it 'returns an Integer' do + described_class.write('foo', '10') + + expect(described_class.read_integer('foo')).to eq(10) + end + + it 'returns nil if no value was found' do + expect(described_class.read_integer('foo')).to be_nil + end + end + + describe '.write' do + it 'writes a value to the cache and returns the written value' do + expect(described_class.write('foo', 10)).to eq(10) + expect(described_class.read('foo')).to eq('10') + end + end + + describe '.set_add' do + it 'adds a value to a set' do + described_class.set_add('foo', 10) + described_class.set_add('foo', 10) + + key = described_class.cache_key_for('foo') + values = Gitlab::Redis::Cache.with { |r| r.smembers(key) } + + expect(values).to eq(['10']) + end + end + + describe '.set_includes?' do + it 'returns false when the key does not exist' do + expect(described_class.set_includes?('foo', 10)).to eq(false) + end + + it 'returns false when the value is not present in the set' do + described_class.set_add('foo', 10) + + expect(described_class.set_includes?('foo', 20)).to eq(false) + end + + it 'returns true when the set includes the given value' do + described_class.set_add('foo', 10) + + expect(described_class.set_includes?('foo', 10)).to eq(true) + end + end + + describe '.write_multiple' do + it 'sets multiple keys' do + mapping = { 'foo' => 10, 'bar' => 20 } + + described_class.write_multiple(mapping) + + mapping.each do |key, value| + full_key = described_class.cache_key_for(key) + found = Gitlab::Redis::Cache.with { |r| r.get(full_key) } + + expect(found).to eq(value.to_s) + end + end + end + + describe '.expire' do + it 'sets the expiration time of a key' do + timeout = 1.hour.to_i + + described_class.write('foo', 'bar', timeout: 2.hours.to_i) + described_class.expire('foo', timeout) + + key = described_class.cache_key_for('foo') + found_ttl = Gitlab::Redis::Cache.with { |r| r.ttl(key) } + + expect(found_ttl).to be <= timeout + end + end +end diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 66273255b6f..6bbdf6bf3b6 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -1,97 +1,286 @@ require 'spec_helper' describe Gitlab::GithubImport::Client do - let(:token) { '123456' } - let(:github_provider) { Settingslogic.new('app_id' => 'asd123', 'app_secret' => 'asd123', 'name' => 'github', 'args' => { 'client_options' => {} }) } + describe '#parallel?' do + it 'returns true when the client is running in parallel mode' do + client = described_class.new('foo', parallel: true) - subject(:client) { described_class.new(token) } + expect(client).to be_parallel + end + + it 'returns false when the client is running in sequential mode' do + client = described_class.new('foo', parallel: false) - before do - allow(Gitlab.config.omniauth).to receive(:providers).and_return([github_provider]) + expect(client).not_to be_parallel + end end - it 'convert OAuth2 client options to symbols' do - client.client.options.keys.each do |key| - expect(key).to be_kind_of(Symbol) + describe '#user' do + it 'returns the details for the given username' do + client = described_class.new('foo') + + expect(client.octokit).to receive(:user).with('foo') + expect(client).to receive(:with_rate_limit).and_yield + + client.user('foo') end end - it 'does not crash (e.g. Settingslogic::MissingSetting) when verify_ssl config is not present' do - expect { client.api }.not_to raise_error + describe '#repository' do + it 'returns the details of a repository' do + client = described_class.new('foo') + + expect(client.octokit).to receive(:repo).with('foo/bar') + expect(client).to receive(:with_rate_limit).and_yield + + client.repository('foo/bar') + end end - context 'when config is missing' do - before do - allow(Gitlab.config.omniauth).to receive(:providers).and_return([]) + describe '#labels' do + it 'returns the labels' do + client = described_class.new('foo') + + expect(client) + .to receive(:each_object) + .with(:labels, 'foo/bar') + + client.labels('foo/bar') end + end + + describe '#milestones' do + it 'returns the milestones' do + client = described_class.new('foo') + + expect(client) + .to receive(:each_object) + .with(:milestones, 'foo/bar') - it 'is still possible to get an Octokit client' do - expect { client.api }.not_to raise_error + client.milestones('foo/bar') end + end + + describe '#releases' do + it 'returns the releases' do + client = described_class.new('foo') - it 'is not be possible to get an OAuth2 client' do - expect { client.client }.to raise_error(Projects::ImportService::Error) + expect(client) + .to receive(:each_object) + .with(:releases, 'foo/bar') + + client.releases('foo/bar') end end - context 'allow SSL verification to be configurable on API' do + describe '#each_page' do + let(:client) { described_class.new('foo') } + let(:object1) { double(:object1) } + let(:object2) { double(:object2) } + before do - github_provider['verify_ssl'] = false - end + allow(client) + .to receive(:with_rate_limit) + .and_yield + + allow(client.octokit) + .to receive(:public_send) + .and_return([object1]) + + response = double(:response, data: [object2], rels: { next: nil }) + next_page = double(:next_page, get: response) - it 'uses supplied value' do - expect(client.client.options[:connection_opts][:ssl]).to eq({ verify: false }) - expect(client.api.connection_options[:ssl]).to eq({ verify: false }) + allow(client.octokit) + .to receive(:last_response) + .and_return(double(:last_response, rels: { next: next_page })) end - end - describe '#api_endpoint' do - context 'when provider does not specity an API endpoint' do - it 'uses GitHub root API endpoint' do - expect(client.api.api_endpoint).to eq 'https://api.github.com/' + context 'without a block' do + it 'returns an Enumerator' do + expect(client.each_page(:foo)).to be_an_instance_of(Enumerator) + end + + it 'the returned Enumerator returns Page objects' do + enum = client.each_page(:foo) + + page1 = enum.next + page2 = enum.next + + expect(page1).to be_an_instance_of(described_class::Page) + expect(page2).to be_an_instance_of(described_class::Page) + + expect(page1.objects).to eq([object1]) + expect(page1.number).to eq(1) + + expect(page2.objects).to eq([object2]) + expect(page2.number).to eq(2) end end - context 'when provider specify a custom API endpoint' do - before do - github_provider['args']['client_options']['site'] = 'https://github.company.com/' + context 'with a block' do + it 'yields every retrieved page to the supplied block' do + pages = [] + + client.each_page(:foo) { |page| pages << page } + + expect(pages[0]).to be_an_instance_of(described_class::Page) + expect(pages[1]).to be_an_instance_of(described_class::Page) + + expect(pages[0].objects).to eq([object1]) + expect(pages[0].number).to eq(1) + + expect(pages[1].objects).to eq([object2]) + expect(pages[1].number).to eq(2) end - it 'uses the custom API endpoint' do - expect(OmniAuth::Strategies::GitHub).not_to receive(:default_options) - expect(client.api.api_endpoint).to eq 'https://github.company.com/' + it 'starts at the given page' do + pages = [] + + client.each_page(:foo, page: 2) { |page| pages << page } + + expect(pages[0].number).to eq(2) + expect(pages[1].number).to eq(3) end end + end - context 'when given a host' do - subject(:client) { described_class.new(token, host: 'https://try.gitea.io/') } + describe '#with_rate_limit' do + let(:client) { described_class.new('foo') } - it 'builds a endpoint with the given host and the default API version' do - expect(client.api.api_endpoint).to eq 'https://try.gitea.io/api/v3/' - end + it 'yields the supplied block when enough requests remain' do + expect(client).to receive(:requests_remaining?).and_return(true) + + expect { |b| client.with_rate_limit(&b) }.to yield_control end - context 'when given an API version' do - subject(:client) { described_class.new(token, api_version: 'v3') } + it 'waits before yielding if not enough requests remain' do + expect(client).to receive(:requests_remaining?).and_return(false) + expect(client).to receive(:raise_or_wait_for_rate_limit) - it 'does not use the API version without a host' do - expect(client.api.api_endpoint).to eq 'https://api.github.com/' - end + expect { |b| client.with_rate_limit(&b) }.to yield_control end - context 'when given a host and version' do - subject(:client) { described_class.new(token, host: 'https://try.gitea.io/', api_version: 'v3') } + it 'waits and retries the operation if all requests were consumed in the supplied block' do + retries = 0 + + expect(client).to receive(:requests_remaining?).and_return(true) + expect(client).to receive(:raise_or_wait_for_rate_limit) - it 'builds a endpoint with the given options' do - expect(client.api.api_endpoint).to eq 'https://try.gitea.io/api/v3/' + client.with_rate_limit do + if retries.zero? + retries += 1 + raise(Octokit::TooManyRequests) + end end + + expect(retries).to eq(1) + end + + it 'increments the request count counter' do + expect(client.request_count_counter) + .to receive(:increment) + .and_call_original + + expect(client).to receive(:requests_remaining?).and_return(true) + + client.with_rate_limit { } + end + end + + describe '#requests_remaining?' do + let(:client) { described_class.new('foo') } + + it 'returns true if enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(9000) + + expect(client.requests_remaining?).to eq(true) + end + + it 'returns false if not enough requests remain' do + expect(client).to receive(:remaining_requests).and_return(1) + + expect(client.requests_remaining?).to eq(false) + end + end + + describe '#raise_or_wait_for_rate_limit' do + it 'raises RateLimitError when running in parallel mode' do + client = described_class.new('foo', parallel: true) + + expect { client.raise_or_wait_for_rate_limit } + .to raise_error(Gitlab::GithubImport::RateLimitError) + end + + it 'sleeps when running in sequential mode' do + client = described_class.new('foo', parallel: false) + + expect(client).to receive(:rate_limit_resets_in).and_return(1) + expect(client).to receive(:sleep).with(1) + + client.raise_or_wait_for_rate_limit + end + + it 'increments the rate limit counter' do + client = described_class.new('foo', parallel: false) + + expect(client) + .to receive(:rate_limit_resets_in) + .and_return(1) + + expect(client) + .to receive(:sleep) + .with(1) + + expect(client.rate_limit_counter) + .to receive(:increment) + .and_call_original + + client.raise_or_wait_for_rate_limit end end - it 'does not raise error when rate limit is disabled' do - stub_request(:get, /api.github.com/) - allow(client.api).to receive(:rate_limit!).and_raise(Octokit::NotFound) + describe '#remaining_requests' do + it 'returns the number of remaining requests' do + client = described_class.new('foo') + rate_limit = double(remaining: 1) + + expect(client.octokit).to receive(:rate_limit).and_return(rate_limit) + expect(client.remaining_requests).to eq(1) + end + end + + describe '#rate_limit_resets_in' do + it 'returns the number of seconds after which the rate limit is reset' do + client = described_class.new('foo') + rate_limit = double(resets_in: 1) + + expect(client.octokit).to receive(:rate_limit).and_return(rate_limit) + + expect(client.rate_limit_resets_in).to eq(6) + end + end - expect { client.issues {} }.not_to raise_error + describe '#method_missing' do + it 'delegates missing methods to the request method' do + client = described_class.new('foo') + + expect(client).to receive(:milestones).with(state: 'all') + + client.milestones(state: 'all') + end + end + + describe '#respond_to_missing?' do + it 'returns true for methods supported by Octokit' do + client = described_class.new('foo') + + expect(client.respond_to?(:milestones)).to eq(true) + end + + it 'returns false for methods not supported by Octokit' do + client = described_class.new('foo') + + expect(client.respond_to?(:kittens)).to eq(false) + end end end diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb new file mode 100644 index 00000000000..1568c657a1e --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -0,0 +1,152 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::DiffNoteImporter do + let(:project) { create(:project) } + let(:client) { double(:client) } + let(:user) { create(:user) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:hunk) do + '@@ -1 +1 @@ + -Hello + +Hello world' + end + + let(:note) do + Gitlab::GithubImport::Representation::DiffNote.new( + noteable_type: 'MergeRequest', + noteable_id: 1, + commit_id: '123abc', + file_path: 'README.md', + diff_hunk: hunk, + author: Gitlab::GithubImport::Representation::User + .new(id: user.id, login: user.username), + note: 'Hello', + created_at: created_at, + updated_at: updated_at, + github_id: 1 + ) + end + + let(:importer) { described_class.new(note, project, client) } + + describe '#execute' do + context 'when the merge request no longer exists' do + it 'does not import anything' do + expect(Gitlab::Database).not_to receive(:bulk_insert) + + importer.execute + end + end + + context 'when the merge request exists' do + let!(:merge_request) do + create(:merge_request, source_project: project, target_project: project) + end + + before do + allow(importer) + .to receive(:find_merge_request_id) + .and_return(merge_request.id) + end + + it 'imports the note' do + allow(importer.user_finder) + .to receive(:author_id_for) + .and_return([user.id, true]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + LegacyDiffNote.table_name, + [ + { + noteable_type: 'MergeRequest', + noteable_id: merge_request.id, + project_id: project.id, + author_id: user.id, + note: 'Hello', + system: false, + commit_id: '123abc', + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: created_at, + updated_at: updated_at, + st_diff: note.diff_hash.to_yaml + } + ] + ) + .and_call_original + + importer.execute + end + + it 'imports the note when the author could not be found' do + allow(importer.user_finder) + .to receive(:author_id_for) + .and_return([project.creator_id, false]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + LegacyDiffNote.table_name, + [ + { + noteable_type: 'MergeRequest', + noteable_id: merge_request.id, + project_id: project.id, + author_id: project.creator_id, + note: "*Created by: #{user.username}*\n\nHello", + system: false, + commit_id: '123abc', + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: created_at, + updated_at: updated_at, + st_diff: note.diff_hash.to_yaml + } + ] + ) + .and_call_original + + importer.execute + end + + it 'produces a valid LegacyDiffNote' do + allow(importer.user_finder) + .to receive(:author_id_for) + .and_return([user.id, true]) + + importer.execute + + note = project.notes.diff_notes.take + + expect(note).to be_valid + expect(note.diff).to be_an_instance_of(Gitlab::Git::Diff) + end + + it 'does not import the note when a foreign key error is raised' do + allow(importer.user_finder) + .to receive(:author_id_for) + .and_return([project.creator_id, false]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') + + expect { importer.execute }.not_to raise_error + end + end + end + + describe '#find_merge_request_id' do + it 'returns a merge request ID' do + expect_any_instance_of(Gitlab::GithubImport::IssuableFinder) + .to receive(:database_id) + .and_return(10) + + expect(importer.find_merge_request_id).to eq(10) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb new file mode 100644 index 00000000000..4713c6795bb --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb @@ -0,0 +1,119 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::DiffNotesImporter do + let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let(:client) { double(:client) } + + let(:github_comment) do + double( + :response, + html_url: 'https://github.com/foo/bar/pull/42', + path: 'README.md', + commit_id: '123abc', + diff_hunk: "@@ -1 +1 @@\n-Hello\n+Hello world", + user: double(:user, id: 4, login: 'alice'), + body: 'Hello world', + created_at: Time.zone.now, + updated_at: Time.zone.now, + id: 1 + ) + end + + describe '#parallel?' do + it 'returns true when running in parallel mode' do + importer = described_class.new(project, client) + expect(importer).to be_parallel + end + + it 'returns false when running in sequential mode' do + importer = described_class.new(project, client, parallel: false) + expect(importer).not_to be_parallel + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports diff notes in parallel' do + importer = described_class.new(project, client) + + expect(importer).to receive(:parallel_import) + + importer.execute + end + end + + context 'when running in sequential mode' do + it 'imports diff notes in sequence' do + importer = described_class.new(project, client, parallel: false) + + expect(importer).to receive(:sequential_import) + + importer.execute + end + end + end + + describe '#sequential_import' do + it 'imports each diff note in sequence' do + importer = described_class.new(project, client, parallel: false) + diff_note_importer = double(:diff_note_importer) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_comment) + + expect(Gitlab::GithubImport::Importer::DiffNoteImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::DiffNote), + project, + client + ) + .and_return(diff_note_importer) + + expect(diff_note_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import' do + it 'imports each diff note in parallel' do + importer = described_class.new(project, client) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_comment) + + expect(Gitlab::GithubImport::ImportDiffNoteWorker) + .to receive(:perform_async) + .with(project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the ID of the given note' do + importer = described_class.new(project, client) + + expect(importer.id_for_already_imported_cache(github_comment)) + .to eq(1) + end + end + + describe '#collection_options' do + it 'returns an empty Hash' do + # For large projects (e.g. kubernetes/kubernetes) GitHub's API may produce + # HTTP 500 errors when using explicit sorting options, regardless of what + # order you sort in. Not using any sorting options at all allows us to + # work around this. + importer = described_class.new(project, client) + + expect(importer.collection_options).to eq({}) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/issue_and_label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_and_label_links_importer_spec.rb new file mode 100644 index 00000000000..665b31ef244 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/issue_and_label_links_importer_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::IssueAndLabelLinksImporter do + describe '#execute' do + it 'imports an issue and its labels' do + issue = double(:issue) + project = double(:project) + client = double(:client) + label_links_instance = double(:label_links_importer) + importer = described_class.new(issue, project, client) + + expect(Gitlab::GithubImport::Importer::IssueImporter) + .to receive(:import_if_issue) + .with(issue, project, client) + + expect(Gitlab::GithubImport::Importer::LabelLinksImporter) + .to receive(:new) + .with(issue, project, client) + .and_return(label_links_instance) + + expect(label_links_instance) + .to receive(:execute) + + importer.execute + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb new file mode 100644 index 00000000000..d34ca0b76b8 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb @@ -0,0 +1,201 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cache do + let(:project) { create(:project) } + let(:client) { double(:client) } + let(:user) { create(:user) } + let(:milestone) { create(:milestone, project: project) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:issue) do + Gitlab::GithubImport::Representation::Issue.new( + iid: 42, + title: 'My Issue', + description: 'This is my issue', + milestone_number: 1, + state: :opened, + assignees: [ + Gitlab::GithubImport::Representation::User.new(id: 4, login: 'alice'), + Gitlab::GithubImport::Representation::User.new(id: 5, login: 'bob') + ], + label_names: %w[bug], + author: Gitlab::GithubImport::Representation::User.new(id: 4, login: 'alice'), + created_at: created_at, + updated_at: updated_at, + pull_request: false + ) + end + + describe '.import_if_issue' do + it 'imports an issuable if it is a regular issue' do + importer = double(:importer) + + expect(described_class) + .to receive(:new) + .with(issue, project, client) + .and_return(importer) + + expect(importer).to receive(:execute) + + described_class.import_if_issue(issue, project, client) + end + + it 'does not import the issuable if it is a pull request' do + expect(issue).to receive(:pull_request?).and_return(true) + + expect(described_class).not_to receive(:new) + + described_class.import_if_issue(issue, project, client) + end + end + + describe '#execute' do + let(:importer) { described_class.new(issue, project, client) } + + it 'creates the issue and assignees' do + expect(importer) + .to receive(:create_issue) + .and_return(10) + + expect(importer) + .to receive(:create_assignees) + .with(10) + + expect(importer.issuable_finder) + .to receive(:cache_database_id) + .with(10) + + importer.execute + end + end + + describe '#create_issue' do + let(:importer) { described_class.new(issue, project, client) } + + before do + allow(importer.milestone_finder) + .to receive(:id_for) + .with(issue) + .and_return(milestone.id) + end + + context 'when the issue author could be found' do + it 'creates the issue with the found author as the issue author' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(issue) + .and_return([user.id, true]) + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .with( + { + iid: 42, + title: 'My Issue', + author_id: user.id, + project_id: project.id, + description: 'This is my issue', + milestone_id: milestone.id, + state: :opened, + created_at: created_at, + updated_at: updated_at + }, + project.issues + ) + .and_call_original + + importer.create_issue + end + end + + context 'when the issue author could not be found' do + it 'creates the issue with the project creator as the issue author' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(issue) + .and_return([project.creator_id, false]) + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .with( + { + iid: 42, + title: 'My Issue', + author_id: project.creator_id, + project_id: project.id, + description: "*Created by: alice*\n\nThis is my issue", + milestone_id: milestone.id, + state: :opened, + created_at: created_at, + updated_at: updated_at + }, + project.issues + ) + .and_call_original + + importer.create_issue + end + end + + context 'when the import fails due to a foreign key error' do + it 'does not raise any errors' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(issue) + .and_return([user.id, true]) + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') + + expect { importer.create_issue }.not_to raise_error + end + end + + it 'produces a valid Issue' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(issue) + .and_return([user.id, true]) + + importer.create_issue + + expect(project.issues.take).to be_valid + end + + it 'returns the ID of the created issue' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(issue) + .and_return([user.id, true]) + + expect(importer.create_issue).to be_a_kind_of(Numeric) + end + end + + describe '#create_assignees' do + it 'inserts the issue assignees in bulk' do + importer = described_class.new(issue, project, client) + + allow(importer.user_finder) + .to receive(:user_id_for) + .ordered.with(issue.assignees[0]) + .and_return(4) + + allow(importer.user_finder) + .to receive(:user_id_for) + .ordered.with(issue.assignees[1]) + .and_return(5) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + IssueAssignee.table_name, + [{ issue_id: 1, user_id: 4 }, { issue_id: 1, user_id: 5 }] + ) + + importer.create_assignees(1) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb new file mode 100644 index 00000000000..e237e79e94b --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb @@ -0,0 +1,111 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::IssuesImporter do + let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let(:client) { double(:client) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:github_issue) do + double( + :response, + number: 42, + title: 'My Issue', + body: 'This is my issue', + milestone: double(:milestone, number: 4), + state: 'open', + assignees: [double(:user, id: 4, login: 'alice')], + labels: [double(:label, name: 'bug')], + user: double(:user, id: 4, login: 'alice'), + created_at: created_at, + updated_at: updated_at, + pull_request: false + ) + end + + describe '#parallel?' do + it 'returns true when running in parallel mode' do + importer = described_class.new(project, client) + expect(importer).to be_parallel + end + + it 'returns false when running in sequential mode' do + importer = described_class.new(project, client, parallel: false) + expect(importer).not_to be_parallel + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports issues in parallel' do + importer = described_class.new(project, client) + + expect(importer).to receive(:parallel_import) + + importer.execute + end + end + + context 'when running in sequential mode' do + it 'imports issues in sequence' do + importer = described_class.new(project, client, parallel: false) + + expect(importer).to receive(:sequential_import) + + importer.execute + end + end + end + + describe '#sequential_import' do + it 'imports each issue in sequence' do + importer = described_class.new(project, client, parallel: false) + issue_importer = double(:importer) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_issue) + + expect(Gitlab::GithubImport::Importer::IssueAndLabelLinksImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Issue), + project, + client + ) + .and_return(issue_importer) + + expect(issue_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import' do + it 'imports each issue in parallel' do + importer = described_class.new(project, client) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_issue) + + expect(Gitlab::GithubImport::ImportIssueWorker) + .to receive(:perform_async) + .with(project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the issue number of the given issue' do + importer = described_class.new(project, client) + + expect(importer.id_for_already_imported_cache(github_issue)) + .to eq(42) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb new file mode 100644 index 00000000000..e2a71e78574 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::LabelLinksImporter do + let(:project) { create(:project) } + let(:client) { double(:client) } + let(:issue) do + double( + :issue, + iid: 4, + label_names: %w[bug], + issuable_type: Issue, + pull_request?: false + ) + end + + let(:importer) { described_class.new(issue, project, client) } + + describe '#execute' do + it 'creates the label links' do + importer = described_class.new(issue, project, client) + + expect(importer).to receive(:create_labels) + + importer.execute + end + end + + describe '#create_labels' do + it 'inserts the label links in bulk' do + expect(importer.label_finder) + .to receive(:id_for) + .with('bug') + .and_return(2) + + expect(importer) + .to receive(:find_target_id) + .and_return(1) + + Timecop.freeze do + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + LabelLink.table_name, + [ + { + label_id: 2, + target_id: 1, + target_type: Issue, + created_at: Time.zone.now, + updated_at: Time.zone.now + } + ] + ) + + importer.create_labels + end + end + + it 'does not insert label links for non-existing labels' do + expect(importer.label_finder) + .to receive(:id_for) + .with('bug') + .and_return(nil) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with(LabelLink.table_name, []) + + importer.create_labels + end + end + + describe '#find_target_id' do + it 'returns the ID of the issuable to create the label link for' do + expect_any_instance_of(Gitlab::GithubImport::IssuableFinder) + .to receive(:database_id) + .and_return(10) + + expect(importer.find_target_id).to eq(10) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb new file mode 100644 index 00000000000..156ef96a0fa --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/labels_importer_spec.rb @@ -0,0 +1,107 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::LabelsImporter, :clean_gitlab_redis_cache do + let(:project) { create(:project, import_source: 'foo/bar') } + let(:client) { double(:client) } + let(:importer) { described_class.new(project, client) } + + describe '#execute' do + it 'imports the labels in bulk' do + label_hash = { title: 'bug', color: '#fffaaa' } + + expect(importer) + .to receive(:build_labels) + .and_return([label_hash]) + + expect(importer) + .to receive(:bulk_insert) + .with(Label, [label_hash]) + + expect(importer) + .to receive(:build_labels_cache) + + importer.execute + end + end + + describe '#build_labels' do + it 'returns an Array containnig label rows' do + label = double(:label, name: 'bug', color: 'ffffff') + + expect(importer).to receive(:each_label).and_return([label]) + + rows = importer.build_labels + + expect(rows.length).to eq(1) + expect(rows[0][:title]).to eq('bug') + end + + it 'does not create labels that already exist' do + create(:label, project: project, title: 'bug') + + label = double(:label, name: 'bug', color: 'ffffff') + + expect(importer).to receive(:each_label).and_return([label]) + expect(importer.build_labels).to be_empty + end + end + + describe '#build_labels_cache' do + it 'builds the labels cache' do + expect_any_instance_of(Gitlab::GithubImport::LabelFinder) + .to receive(:build_cache) + + importer.build_labels_cache + end + end + + describe '#build' do + let(:label_hash) do + importer.build(double(:label, name: 'bug', color: 'ffffff')) + end + + it 'returns the attributes of the label as a Hash' do + expect(label_hash).to be_an_instance_of(Hash) + end + + context 'the returned Hash' do + it 'includes the label title' do + expect(label_hash[:title]).to eq('bug') + end + + it 'includes the label color' do + expect(label_hash[:color]).to eq('#ffffff') + end + + it 'includes the project ID' do + expect(label_hash[:project_id]).to eq(project.id) + end + + it 'includes the label type' do + expect(label_hash[:type]).to eq('ProjectLabel') + end + + it 'includes the created timestamp' do + Timecop.freeze do + expect(label_hash[:created_at]).to eq(Time.zone.now) + end + end + + it 'includes the updated timestamp' do + Timecop.freeze do + expect(label_hash[:updated_at]).to eq(Time.zone.now) + end + end + end + end + + describe '#each_label' do + it 'returns the labels' do + expect(client) + .to receive(:labels) + .with('foo/bar') + + importer.each_label + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb new file mode 100644 index 00000000000..b1cac3b6e46 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/milestones_importer_spec.rb @@ -0,0 +1,120 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::MilestonesImporter, :clean_gitlab_redis_cache do + let(:project) { create(:project, import_source: 'foo/bar') } + let(:client) { double(:client) } + let(:importer) { described_class.new(project, client) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:milestone) do + double( + :milestone, + number: 1, + title: '1.0', + description: 'The first release', + state: 'open', + created_at: created_at, + updated_at: updated_at + ) + end + + describe '#execute' do + it 'imports the milestones in bulk' do + milestone_hash = { number: 1, title: '1.0' } + + expect(importer) + .to receive(:build_milestones) + .and_return([milestone_hash]) + + expect(importer) + .to receive(:bulk_insert) + .with(Milestone, [milestone_hash]) + + expect(importer) + .to receive(:build_milestones_cache) + + importer.execute + end + end + + describe '#build_milestones' do + it 'returns an Array containnig milestone rows' do + expect(importer) + .to receive(:each_milestone) + .and_return([milestone]) + + rows = importer.build_milestones + + expect(rows.length).to eq(1) + expect(rows[0][:title]).to eq('1.0') + end + + it 'does not create milestones that already exist' do + create(:milestone, project: project, title: '1.0', iid: 1) + + expect(importer) + .to receive(:each_milestone) + .and_return([milestone]) + + expect(importer.build_milestones).to be_empty + end + end + + describe '#build_milestones_cache' do + it 'builds the milestones cache' do + expect_any_instance_of(Gitlab::GithubImport::MilestoneFinder) + .to receive(:build_cache) + + importer.build_milestones_cache + end + end + + describe '#build' do + let(:milestone_hash) { importer.build(milestone) } + + it 'returns the attributes of the milestone as a Hash' do + expect(milestone_hash).to be_an_instance_of(Hash) + end + + context 'the returned Hash' do + it 'includes the milestone number' do + expect(milestone_hash[:iid]).to eq(1) + end + + it 'includes the milestone title' do + expect(milestone_hash[:title]).to eq('1.0') + end + + it 'includes the milestone description' do + expect(milestone_hash[:description]).to eq('The first release') + end + + it 'includes the project ID' do + expect(milestone_hash[:project_id]).to eq(project.id) + end + + it 'includes the milestone state' do + expect(milestone_hash[:state]).to eq(:active) + end + + it 'includes the created timestamp' do + expect(milestone_hash[:created_at]).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(milestone_hash[:updated_at]).to eq(updated_at) + end + end + end + + describe '#each_milestone' do + it 'returns the milestones' do + expect(client) + .to receive(:milestones) + .with('foo/bar', state: 'all') + + importer.each_milestone + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb new file mode 100644 index 00000000000..9bdcc42be19 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -0,0 +1,151 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::NoteImporter do + let(:client) { double(:client) } + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:github_note) do + Gitlab::GithubImport::Representation::Note.new( + noteable_id: 1, + noteable_type: 'Issue', + author: Gitlab::GithubImport::Representation::User.new(id: 4, login: 'alice'), + note: 'This is my note', + created_at: created_at, + updated_at: updated_at, + github_id: 1 + ) + end + + let(:importer) { described_class.new(github_note, project, client) } + + describe '#execute' do + context 'when the noteable exists' do + let!(:issue_row) { create(:issue, project: project, iid: 1) } + + before do + allow(importer) + .to receive(:find_noteable_id) + .and_return(issue_row.id) + end + + context 'when the author could be found' do + it 'imports the note with the found author as the note author' do + expect(importer.user_finder) + .to receive(:author_id_for) + .with(github_note) + .and_return([user.id, true]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + Note.table_name, + [ + { + noteable_type: 'Issue', + noteable_id: issue_row.id, + project_id: project.id, + author_id: user.id, + note: 'This is my note', + system: false, + created_at: created_at, + updated_at: updated_at + } + ] + ) + .and_call_original + + importer.execute + end + end + + context 'when the note author could not be found' do + it 'imports the note with the project creator as the note author' do + expect(importer.user_finder) + .to receive(:author_id_for) + .with(github_note) + .and_return([project.creator_id, false]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with( + Note.table_name, + [ + { + noteable_type: 'Issue', + noteable_id: issue_row.id, + project_id: project.id, + author_id: project.creator_id, + note: "*Created by: alice*\n\nThis is my note", + system: false, + created_at: created_at, + updated_at: updated_at + } + ] + ) + .and_call_original + + importer.execute + end + end + end + + context 'when the noteable does not exist' do + it 'does not import the note' do + expect(Gitlab::Database).not_to receive(:bulk_insert) + + importer.execute + end + end + + context 'when the import fails due to a foreign key error' do + it 'does not raise any errors' do + issue_row = create(:issue, project: project, iid: 1) + + allow(importer) + .to receive(:find_noteable_id) + .and_return(issue_row.id) + + allow(importer.user_finder) + .to receive(:author_id_for) + .with(github_note) + .and_return([user.id, true]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') + + expect { importer.execute }.not_to raise_error + end + end + + it 'produces a valid Note' do + issue_row = create(:issue, project: project, iid: 1) + + allow(importer) + .to receive(:find_noteable_id) + .and_return(issue_row.id) + + allow(importer.user_finder) + .to receive(:author_id_for) + .with(github_note) + .and_return([user.id, true]) + + importer.execute + + expect(project.notes.take).to be_valid + end + end + + describe '#find_noteable_id' do + it 'returns the ID of the noteable' do + expect_any_instance_of(Gitlab::GithubImport::IssuableFinder) + .to receive(:database_id) + .and_return(10) + + expect(importer.find_noteable_id).to eq(10) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb new file mode 100644 index 00000000000..f046d13f879 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb @@ -0,0 +1,116 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::NotesImporter do + let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let(:client) { double(:client) } + + let(:github_comment) do + double( + :response, + html_url: 'https://github.com/foo/bar/issues/42', + user: double(:user, id: 4, login: 'alice'), + body: 'Hello world', + created_at: Time.zone.now, + updated_at: Time.zone.now, + id: 1 + ) + end + + describe '#parallel?' do + it 'returns true when running in parallel mode' do + importer = described_class.new(project, client) + expect(importer).to be_parallel + end + + it 'returns false when running in sequential mode' do + importer = described_class.new(project, client, parallel: false) + expect(importer).not_to be_parallel + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports notes in parallel' do + importer = described_class.new(project, client) + + expect(importer).to receive(:parallel_import) + + importer.execute + end + end + + context 'when running in sequential mode' do + it 'imports notes in sequence' do + importer = described_class.new(project, client, parallel: false) + + expect(importer).to receive(:sequential_import) + + importer.execute + end + end + end + + describe '#sequential_import' do + it 'imports each note in sequence' do + importer = described_class.new(project, client, parallel: false) + note_importer = double(:note_importer) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_comment) + + expect(Gitlab::GithubImport::Importer::NoteImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Note), + project, + client + ) + .and_return(note_importer) + + expect(note_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import' do + it 'imports each note in parallel' do + importer = described_class.new(project, client) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(github_comment) + + expect(Gitlab::GithubImport::ImportNoteWorker) + .to receive(:perform_async) + .with(project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the ID of the given note' do + importer = described_class.new(project, client) + + expect(importer.id_for_already_imported_cache(github_comment)) + .to eq(1) + end + end + + describe '#collection_options' do + it 'returns an empty Hash' do + # For large projects (e.g. kubernetes/kubernetes) GitHub's API may produce + # HTTP 500 errors when using explicit sorting options, regardless of what + # order you sort in. Not using any sorting options at all allows us to + # work around this. + importer = described_class.new(project, client) + + expect(importer.collection_options).to eq({}) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb new file mode 100644 index 00000000000..35f3fdf8304 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb @@ -0,0 +1,221 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redis_cache do + let(:project) { create(:project, :repository) } + let(:client) { double(:client) } + let(:user) { create(:user) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + let(:merged_at) { Time.new(2017, 1, 1, 12, 17) } + + let(:source_commit) { project.repository.commit('feature') } + let(:target_commit) { project.repository.commit('master') } + let(:milestone) { create(:milestone, project: project) } + + let(:pull_request) do + alice = Gitlab::GithubImport::Representation::User.new(id: 4, login: 'alice') + + Gitlab::GithubImport::Representation::PullRequest.new( + iid: 42, + title: 'My Pull Request', + description: 'This is my pull request', + source_branch: 'feature', + source_branch_sha: source_commit.id, + target_branch: 'master', + target_branch_sha: target_commit.id, + source_repository_id: 400, + target_repository_id: 200, + source_repository_owner: 'alice', + state: :closed, + milestone_number: milestone.iid, + author: alice, + assignee: alice, + created_at: created_at, + updated_at: updated_at, + merged_at: merged_at + ) + end + + let(:importer) { described_class.new(pull_request, project, client) } + + describe '#execute' do + it 'imports the pull request' do + expect(importer) + .to receive(:create_merge_request) + .and_return(10) + + expect_any_instance_of(Gitlab::GithubImport::IssuableFinder) + .to receive(:cache_database_id) + .with(10) + + importer.execute + end + end + + describe '#create_merge_request' do + before do + allow(importer.milestone_finder) + .to receive(:id_for) + .with(pull_request) + .and_return(milestone.id) + end + + context 'when the author could be found' do + before do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(pull_request) + .and_return([user.id, true]) + + allow(importer.user_finder) + .to receive(:assignee_id_for) + .with(pull_request) + .and_return(user.id) + end + + it 'imports the pull request with the pull request author as the merge request author' do + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .with( + { + iid: 42, + title: 'My Pull Request', + description: 'This is my pull request', + source_project_id: project.id, + target_project_id: project.id, + source_branch: 'alice:feature', + target_branch: 'master', + state: :merged, + milestone_id: milestone.id, + author_id: user.id, + assignee_id: user.id, + created_at: created_at, + updated_at: updated_at + }, + project.merge_requests + ) + .and_call_original + + importer.create_merge_request + end + + it 'returns the ID of the created merge request' do + id = importer.create_merge_request + + expect(id).to be_a_kind_of(Numeric) + end + + it 'creates the merge request diffs' do + importer.create_merge_request + + mr = project.merge_requests.take + + expect(mr.merge_request_diffs.exists?).to eq(true) + end + end + + context 'when the author could not be found' do + it 'imports the pull request with the project creator as the merge request author' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(pull_request) + .and_return([project.creator_id, false]) + + allow(importer.user_finder) + .to receive(:assignee_id_for) + .with(pull_request) + .and_return(user.id) + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .with( + { + iid: 42, + title: 'My Pull Request', + description: "*Created by: alice*\n\nThis is my pull request", + source_project_id: project.id, + target_project_id: project.id, + source_branch: 'alice:feature', + target_branch: 'master', + state: :merged, + milestone_id: milestone.id, + author_id: project.creator_id, + assignee_id: user.id, + created_at: created_at, + updated_at: updated_at + }, + project.merge_requests + ) + .and_call_original + + importer.create_merge_request + end + end + + context 'when the source and target branch are identical' do + it 'uses a generated source branch name for the merge request' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(pull_request) + .and_return([user.id, true]) + + allow(importer.user_finder) + .to receive(:assignee_id_for) + .with(pull_request) + .and_return(user.id) + + allow(pull_request) + .to receive(:source_repository_id) + .and_return(pull_request.target_repository_id) + + allow(pull_request) + .to receive(:source_branch) + .and_return('master') + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .with( + { + iid: 42, + title: 'My Pull Request', + description: 'This is my pull request', + source_project_id: project.id, + target_project_id: project.id, + source_branch: 'master-42', + target_branch: 'master', + state: :merged, + milestone_id: milestone.id, + author_id: user.id, + assignee_id: user.id, + created_at: created_at, + updated_at: updated_at + }, + project.merge_requests + ) + .and_call_original + + importer.create_merge_request + end + end + + context 'when the import fails due to a foreign key error' do + it 'does not raise any errors' do + allow(importer.user_finder) + .to receive(:author_id_for) + .with(pull_request) + .and_return([user.id, true]) + + allow(importer.user_finder) + .to receive(:assignee_id_for) + .with(pull_request) + .and_return(user.id) + + expect(Gitlab::GithubImport) + .to receive(:insert_and_return_id) + .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') + + expect { importer.create_merge_request }.not_to raise_error + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb new file mode 100644 index 00000000000..d72572cd510 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/pull_requests_importer_spec.rb @@ -0,0 +1,272 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::PullRequestsImporter do + let(:project) { create(:project, import_source: 'foo/bar') } + let(:client) { double(:client) } + + let(:pull_request) do + double( + :response, + number: 42, + title: 'My Pull Request', + body: 'This is my pull request', + state: 'closed', + head: double( + :head, + sha: '123abc', + ref: 'my-feature', + repo: double(:repo, id: 400), + user: double(:user, id: 4, login: 'alice') + ), + base: double( + :base, + sha: '456def', + ref: 'master', + repo: double(:repo, id: 200) + ), + milestone: double(:milestone, number: 4), + user: double(:user, id: 4, login: 'alice'), + assignee: double(:user, id: 4, login: 'alice'), + created_at: Time.zone.now, + updated_at: Time.zone.now, + merged_at: Time.zone.now + ) + end + + describe '#parallel?' do + it 'returns true when running in parallel mode' do + importer = described_class.new(project, client) + expect(importer).to be_parallel + end + + it 'returns false when running in sequential mode' do + importer = described_class.new(project, client, parallel: false) + expect(importer).not_to be_parallel + end + end + + describe '#execute' do + context 'when running in parallel mode' do + it 'imports pull requests in parallel' do + importer = described_class.new(project, client) + + expect(importer).to receive(:parallel_import) + + importer.execute + end + end + + context 'when running in sequential mode' do + it 'imports pull requests in sequence' do + importer = described_class.new(project, client, parallel: false) + + expect(importer).to receive(:sequential_import) + + importer.execute + end + end + end + + describe '#sequential_import' do + it 'imports each pull request in sequence' do + importer = described_class.new(project, client, parallel: false) + pull_request_importer = double(:pull_request_importer) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(pull_request) + + expect(Gitlab::GithubImport::Importer::PullRequestImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::PullRequest), + project, + client + ) + .and_return(pull_request_importer) + + expect(pull_request_importer).to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import' do + it 'imports each note in parallel' do + importer = described_class.new(project, client) + + allow(importer) + .to receive(:each_object_to_import) + .and_yield(pull_request) + + expect(Gitlab::GithubImport::ImportPullRequestWorker) + .to receive(:perform_async) + .with(project.id, an_instance_of(Hash), an_instance_of(String)) + + waiter = importer.parallel_import + + expect(waiter).to be_an_instance_of(Gitlab::JobWaiter) + expect(waiter.jobs_remaining).to eq(1) + end + end + + describe '#each_object_to_import', :clean_gitlab_redis_cache do + let(:importer) { described_class.new(project, client) } + + before do + page = double(:page, objects: [pull_request], number: 1) + + expect(client) + .to receive(:each_page) + .with( + :pull_requests, + 'foo/bar', + { state: 'all', sort: 'created', direction: 'asc', page: 1 } + ) + .and_yield(page) + end + + it 'yields every pull request to the supplied block' do + expect { |b| importer.each_object_to_import(&b) } + .to yield_with_args(pull_request) + end + + it 'updates the repository if a pull request was updated after the last clone' do + expect(importer) + .to receive(:update_repository?) + .with(pull_request) + .and_return(true) + + expect(importer) + .to receive(:update_repository) + + importer.each_object_to_import { } + end + end + + describe '#update_repository' do + it 'updates the repository' do + importer = described_class.new(project, client) + + expect(project.repository) + .to receive(:fetch_remote) + .with('github', forced: false) + + expect(Rails.logger) + .to receive(:info) + .with(an_instance_of(String)) + + expect(importer.repository_updates_counter) + .to receive(:increment) + .with(project: project.path_with_namespace) + .and_call_original + + Timecop.freeze do + importer.update_repository + + expect(project.last_repository_updated_at).to eq(Time.zone.now) + end + end + end + + describe '#update_repository?' do + let(:importer) { described_class.new(project, client) } + + context 'when the pull request was updated after the last update' do + let(:pr) do + double( + :pr, + updated_at: Time.zone.now, + head: double(:head, sha: '123'), + base: double(:base, sha: '456') + ) + end + + before do + allow(project) + .to receive(:last_repository_updated_at) + .and_return(1.year.ago) + end + + it 'returns true when the head SHA is not present' do + expect(importer) + .to receive(:commit_exists?) + .with(pr.head.sha) + .and_return(false) + + expect(importer.update_repository?(pr)).to eq(true) + end + + it 'returns true when the base SHA is not present' do + expect(importer) + .to receive(:commit_exists?) + .with(pr.head.sha) + .and_return(true) + + expect(importer) + .to receive(:commit_exists?) + .with(pr.base.sha) + .and_return(false) + + expect(importer.update_repository?(pr)).to eq(true) + end + + it 'returns false if both the head and base SHAs are present' do + expect(importer) + .to receive(:commit_exists?) + .with(pr.head.sha) + .and_return(true) + + expect(importer) + .to receive(:commit_exists?) + .with(pr.base.sha) + .and_return(true) + + expect(importer.update_repository?(pr)).to eq(false) + end + end + + context 'when the pull request was updated before the last update' do + it 'returns false' do + pr = double(:pr, updated_at: 1.year.ago) + + allow(project) + .to receive(:last_repository_updated_at) + .and_return(Time.zone.now) + + expect(importer.update_repository?(pr)).to eq(false) + end + end + end + + describe '#commit_exists?' do + let(:importer) { described_class.new(project, client) } + + it 'returns true when a commit exists' do + expect(project.repository) + .to receive(:lookup) + .with('123') + .and_return(double(:commit)) + + expect(importer.commit_exists?('123')).to eq(true) + end + + it 'returns false when a commit does not exist' do + expect(project.repository) + .to receive(:lookup) + .with('123') + .and_raise(Rugged::OdbError) + + expect(importer.commit_exists?('123')).to eq(false) + end + end + + describe '#id_for_already_imported_cache' do + it 'returns the PR number of the given PR' do + importer = described_class.new(project, client) + + expect(importer.id_for_already_imported_cache(pull_request)) + .to eq(42) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb new file mode 100644 index 00000000000..23ae026fb14 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/releases_importer_spec.rb @@ -0,0 +1,125 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::ReleasesImporter do + let(:project) { create(:project) } + let(:client) { double(:client) } + let(:importer) { described_class.new(project, client) } + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + let(:release) do + double( + :release, + tag_name: '1.0', + body: 'This is my release', + created_at: created_at, + updated_at: updated_at + ) + end + + describe '#execute' do + it 'imports the releases in bulk' do + release_hash = { + tag_name: '1.0', + description: 'This is my release', + created_at: created_at, + updated_at: updated_at + } + + expect(importer).to receive(:build_releases).and_return([release_hash]) + expect(importer).to receive(:bulk_insert).with(Release, [release_hash]) + + importer.execute + end + end + + describe '#build_releases' do + it 'returns an Array containnig release rows' do + expect(importer).to receive(:each_release).and_return([release]) + + rows = importer.build_releases + + expect(rows.length).to eq(1) + expect(rows[0][:tag]).to eq('1.0') + end + + it 'does not create releases that already exist' do + create(:release, project: project, tag: '1.0', description: '1.0') + + expect(importer).to receive(:each_release).and_return([release]) + expect(importer.build_releases).to be_empty + end + + it 'uses a default release description if none is provided' do + expect(release).to receive(:body).and_return('') + expect(importer).to receive(:each_release).and_return([release]) + + release = importer.build_releases.first + + expect(release[:description]).to eq('Release for tag 1.0') + end + end + + describe '#build' do + let(:release_hash) { importer.build(release) } + + it 'returns the attributes of the release as a Hash' do + expect(release_hash).to be_an_instance_of(Hash) + end + + context 'the returned Hash' do + it 'includes the tag name' do + expect(release_hash[:tag]).to eq('1.0') + end + + it 'includes the release description' do + expect(release_hash[:description]).to eq('This is my release') + end + + it 'includes the project ID' do + expect(release_hash[:project_id]).to eq(project.id) + end + + it 'includes the created timestamp' do + expect(release_hash[:created_at]).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(release_hash[:updated_at]).to eq(updated_at) + end + end + end + + describe '#each_release' do + let(:release) { double(:release) } + + before do + allow(project).to receive(:import_source).and_return('foo/bar') + + allow(client) + .to receive(:releases) + .with('foo/bar') + .and_return([release].to_enum) + end + + it 'returns an Enumerator' do + expect(importer.each_release).to be_an_instance_of(Enumerator) + end + + it 'yields every release to the Enumerator' do + expect(importer.each_release.next).to eq(release) + end + end + + describe '#description_for' do + it 'returns the description when present' do + expect(importer.description_for(release)).to eq(release.body) + end + + it 'returns a generated description when one is not present' do + allow(release).to receive(:body).and_return('') + + expect(importer.description_for(release)).to eq('Release for tag 1.0') + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb new file mode 100644 index 00000000000..80539807711 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb @@ -0,0 +1,264 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Importer::RepositoryImporter do + let(:repository) { double(:repository) } + let(:client) { double(:client) } + + let(:project) do + double( + :project, + import_url: 'foo.git', + import_source: 'foo/bar', + repository_storage_path: 'foo', + disk_path: 'foo', + repository: repository + ) + end + + let(:importer) { described_class.new(project, client) } + let(:shell_adapter) { Gitlab::Shell.new } + + before do + # The method "gitlab_shell" returns a new instance every call, making + # it harder to set expectations. To work around this we'll stub the method + # and return the same instance on every call. + allow(importer).to receive(:gitlab_shell).and_return(shell_adapter) + end + + describe '#import_wiki?' do + it 'returns true if the wiki should be imported' do + repo = double(:repo, has_wiki: true) + + expect(client) + .to receive(:repository) + .with('foo/bar') + .and_return(repo) + + expect(project) + .to receive(:wiki_repository_exists?) + .and_return(false) + + expect(importer.import_wiki?).to eq(true) + end + + it 'returns false if the GitHub wiki is disabled' do + repo = double(:repo, has_wiki: false) + + expect(client) + .to receive(:repository) + .with('foo/bar') + .and_return(repo) + + expect(importer.import_wiki?).to eq(false) + end + + it 'returns false if the wiki has already been imported' do + repo = double(:repo, has_wiki: true) + + expect(client) + .to receive(:repository) + .with('foo/bar') + .and_return(repo) + + expect(project) + .to receive(:wiki_repository_exists?) + .and_return(true) + + expect(importer.import_wiki?).to eq(false) + end + end + + describe '#execute' do + it 'imports the repository and wiki' do + expect(repository) + .to receive(:empty_repo?) + .and_return(true) + + expect(importer) + .to receive(:import_wiki?) + .and_return(true) + + expect(importer) + .to receive(:import_repository) + .and_return(true) + + expect(importer) + .to receive(:import_wiki_repository) + .and_return(true) + + expect(importer) + .to receive(:update_clone_time) + + expect(importer.execute).to eq(true) + end + + it 'does not import the repository if it already exists' do + expect(repository) + .to receive(:empty_repo?) + .and_return(false) + + expect(importer) + .to receive(:import_wiki?) + .and_return(true) + + expect(importer) + .not_to receive(:import_repository) + + expect(importer) + .to receive(:import_wiki_repository) + .and_return(true) + + expect(importer) + .to receive(:update_clone_time) + + expect(importer.execute).to eq(true) + end + + it 'does not import the wiki if it is disabled' do + expect(repository) + .to receive(:empty_repo?) + .and_return(true) + + expect(importer) + .to receive(:import_wiki?) + .and_return(false) + + expect(importer) + .to receive(:import_repository) + .and_return(true) + + expect(importer) + .to receive(:update_clone_time) + + expect(importer) + .not_to receive(:import_wiki_repository) + + expect(importer.execute).to eq(true) + end + + it 'does not import the wiki if the repository could not be imported' do + expect(repository) + .to receive(:empty_repo?) + .and_return(true) + + expect(importer) + .to receive(:import_wiki?) + .and_return(true) + + expect(importer) + .to receive(:import_repository) + .and_return(false) + + expect(importer) + .not_to receive(:update_clone_time) + + expect(importer) + .not_to receive(:import_wiki_repository) + + expect(importer.execute).to eq(false) + end + end + + describe '#import_repository' do + it 'imports the repository' do + expect(project) + .to receive(:ensure_repository) + + expect(importer) + .to receive(:configure_repository_remote) + + expect(repository) + .to receive(:fetch_remote) + .with('github', forced: true) + + expect(importer.import_repository).to eq(true) + end + + it 'marks the import as failed when an error was raised' do + expect(project).to receive(:ensure_repository) + .and_raise(Gitlab::Git::Repository::NoRepository) + + expect(importer) + .to receive(:fail_import) + .and_return(false) + + expect(importer.import_repository).to eq(false) + end + end + + describe '#configure_repository_remote' do + it 'configures the remote details' do + expect(repository) + .to receive(:remote_exists?) + .with('github') + .and_return(false) + + expect(repository) + .to receive(:add_remote) + .with('github', 'foo.git') + + expect(repository) + .to receive(:set_import_remote_as_mirror) + .with('github') + + expect(repository) + .to receive(:add_remote_fetch_config) + + importer.configure_repository_remote + end + + it 'does not configure the remote if already configured' do + expect(repository) + .to receive(:remote_exists?) + .with('github') + .and_return(true) + + expect(repository) + .not_to receive(:add_remote) + + importer.configure_repository_remote + end + end + + describe '#import_wiki_repository' do + it 'imports the wiki repository' do + expect(importer.gitlab_shell) + .to receive(:import_repository) + .with('foo', 'foo.wiki', 'foo.wiki.git') + + expect(importer.import_wiki_repository).to eq(true) + end + + it 'marks the import as failed if an error was raised' do + expect(importer.gitlab_shell) + .to receive(:import_repository) + .and_raise(Gitlab::Shell::Error) + + expect(importer) + .to receive(:fail_import) + .and_return(false) + + expect(importer.import_wiki_repository).to eq(false) + end + end + + describe '#fail_import' do + it 'marks the import as failed' do + expect(project).to receive(:mark_import_as_failed).with('foo') + + expect(importer.fail_import('foo')).to eq(false) + end + end + + describe '#update_clone_time' do + it 'sets the timestamp for when the cloning process finished' do + Timecop.freeze do + expect(project) + .to receive(:update_column) + .with(:last_repository_updated_at, Time.zone.now) + + importer.update_clone_time + end + end + end +end diff --git a/spec/lib/gitlab/github_import/issuable_finder_spec.rb b/spec/lib/gitlab/github_import/issuable_finder_spec.rb new file mode 100644 index 00000000000..da69911812a --- /dev/null +++ b/spec/lib/gitlab/github_import/issuable_finder_spec.rb @@ -0,0 +1,38 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::IssuableFinder, :clean_gitlab_redis_cache do + let(:project) { double(:project, id: 4) } + let(:issue) do + double(:issue, issuable_type: MergeRequest, iid: 1) + end + + let(:finder) { described_class.new(project, issue) } + + describe '#database_id' do + it 'returns nil when no cache is in place' do + expect(finder.database_id).to be_nil + end + + it 'returns the ID of an issuable when the cache is in place' do + finder.cache_database_id(10) + + expect(finder.database_id).to eq(10) + end + + it 'raises TypeError when the object is not supported' do + finder = described_class.new(project, double(:issue)) + + expect { finder.database_id }.to raise_error(TypeError) + end + end + + describe '#cache_database_id' do + it 'caches the ID of a database row' do + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with('github-import/issuable-finder/4/MergeRequest/1', 10) + + finder.cache_database_id(10) + end + end +end diff --git a/spec/lib/gitlab/github_import/label_finder_spec.rb b/spec/lib/gitlab/github_import/label_finder_spec.rb new file mode 100644 index 00000000000..8ba766944d6 --- /dev/null +++ b/spec/lib/gitlab/github_import/label_finder_spec.rb @@ -0,0 +1,61 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::LabelFinder, :clean_gitlab_redis_cache do + let(:project) { create(:project) } + let(:finder) { described_class.new(project) } + let!(:bug) { create(:label, project: project, name: 'Bug') } + let!(:feature) { create(:label, project: project, name: 'Feature') } + + describe '#id_for' do + context 'with a cache in place' do + before do + finder.build_cache + end + + it 'returns the ID of the given label' do + expect(finder.id_for(feature.name)).to eq(feature.id) + end + + it 'returns nil for an empty cache key' do + key = finder.cache_key_for(bug.name) + + Gitlab::GithubImport::Caching.write(key, '') + + expect(finder.id_for(bug.name)).to be_nil + end + + it 'returns nil for a non existing label name' do + expect(finder.id_for('kittens')).to be_nil + end + end + + context 'without a cache in place' do + it 'returns nil for a label' do + expect(finder.id_for(feature.name)).to be_nil + end + end + end + + describe '#build_cache' do + it 'builds the cache of all project labels' do + expect(Gitlab::GithubImport::Caching) + .to receive(:write_multiple) + .with( + { + "github-import/label-finder/#{project.id}/Bug" => bug.id, + "github-import/label-finder/#{project.id}/Feature" => feature.id + } + ) + .and_call_original + + finder.build_cache + end + end + + describe '#cache_key_for' do + it 'returns the cache key for a label name' do + expect(finder.cache_key_for('foo')) + .to eq("github-import/label-finder/#{project.id}/foo") + end + end +end diff --git a/spec/lib/gitlab/github_import/markdown_text_spec.rb b/spec/lib/gitlab/github_import/markdown_text_spec.rb new file mode 100644 index 00000000000..1ff5b9d66b3 --- /dev/null +++ b/spec/lib/gitlab/github_import/markdown_text_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::MarkdownText do + describe '.format' do + it 'formats the text' do + author = double(:author, login: 'Alice') + text = described_class.format('Hello', author) + + expect(text).to eq("*Created by: Alice*\n\nHello") + end + end + + describe '#to_s' do + it 'returns the text when the author was found' do + author = double(:author, login: 'Alice') + text = described_class.new('Hello', author, true) + + expect(text.to_s).to eq('Hello') + end + + it 'returns the text with an extra header when the author was not found' do + author = double(:author, login: 'Alice') + text = described_class.new('Hello', author) + + expect(text.to_s).to eq("*Created by: Alice*\n\nHello") + end + end +end diff --git a/spec/lib/gitlab/github_import/milestone_finder_spec.rb b/spec/lib/gitlab/github_import/milestone_finder_spec.rb new file mode 100644 index 00000000000..dff931a2fe8 --- /dev/null +++ b/spec/lib/gitlab/github_import/milestone_finder_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::MilestoneFinder, :clean_gitlab_redis_cache do + let!(:project) { create(:project) } + let!(:milestone) { create(:milestone, project: project) } + let(:finder) { described_class.new(project) } + + describe '#id_for' do + let(:issuable) { double(:issuable, milestone_number: milestone.iid) } + + context 'with a cache in place' do + before do + finder.build_cache + end + + it 'returns the milestone ID of the given issuable' do + expect(finder.id_for(issuable)).to eq(milestone.id) + end + + it 'returns nil for an empty cache key' do + key = finder.cache_key_for(milestone.iid) + + Gitlab::GithubImport::Caching.write(key, '') + + expect(finder.id_for(issuable)).to be_nil + end + + it 'returns nil for an issuable with a non-existing milestone' do + expect(finder.id_for(double(:issuable, milestone_number: 5))).to be_nil + end + end + + context 'without a cache in place' do + it 'returns nil' do + expect(finder.id_for(issuable)).to be_nil + end + end + end + + describe '#build_cache' do + it 'builds the cache of all project milestones' do + expect(Gitlab::GithubImport::Caching) + .to receive(:write_multiple) + .with("github-import/milestone-finder/#{project.id}/1" => milestone.id) + .and_call_original + + finder.build_cache + end + end + + describe '#cache_key_for' do + it 'returns the cache key for an IID' do + expect(finder.cache_key_for(10)) + .to eq("github-import/milestone-finder/#{project.id}/10") + end + end +end diff --git a/spec/lib/gitlab/github_import/page_counter_spec.rb b/spec/lib/gitlab/github_import/page_counter_spec.rb new file mode 100644 index 00000000000..c2613a9a415 --- /dev/null +++ b/spec/lib/gitlab/github_import/page_counter_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::PageCounter, :clean_gitlab_redis_cache do + let(:project) { double(:project, id: 1) } + let(:counter) { described_class.new(project, :issues) } + + describe '#initialize' do + it 'sets the initial page number to 1 when no value is cached' do + expect(counter.current).to eq(1) + end + + it 'sets the initial page number to the cached value when one is present' do + Gitlab::GithubImport::Caching.write(counter.cache_key, 2) + + expect(described_class.new(project, :issues).current).to eq(2) + end + end + + describe '#set' do + it 'overwrites the page number when the given number is greater than the current number' do + counter.set(4) + expect(counter.current).to eq(4) + end + + it 'does not overwrite the page number when the given number is lower than the current number' do + counter.set(2) + counter.set(1) + + expect(counter.current).to eq(2) + end + end +end diff --git a/spec/lib/gitlab/github_import/parallel_importer_spec.rb b/spec/lib/gitlab/github_import/parallel_importer_spec.rb new file mode 100644 index 00000000000..e2a821d4d5c --- /dev/null +++ b/spec/lib/gitlab/github_import/parallel_importer_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ParallelImporter do + describe '.async?' do + it 'returns true' do + expect(described_class).to be_async + end + end + + describe '#execute', :clean_gitlab_redis_shared_state do + let(:project) { create(:project) } + let(:importer) { described_class.new(project) } + + before do + expect(Gitlab::GithubImport::Stage::ImportRepositoryWorker) + .to receive(:perform_async) + .with(project.id) + .and_return('123') + end + + it 'schedules the importing of the repository' do + expect(importer.execute).to eq(true) + end + + it 'sets the JID in Redis' do + expect(Gitlab::SidekiqStatus) + .to receive(:set) + .with("github-importer/#{project.id}", StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + .and_call_original + + importer.execute + end + + it 'updates the import JID of the project' do + importer.execute + + expect(project.import_jid).to eq("github-importer/#{project.id}") + end + end +end diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb new file mode 100644 index 00000000000..98205d3ee25 --- /dev/null +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -0,0 +1,296 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ParallelScheduling do + let(:importer_class) do + Class.new do + include(Gitlab::GithubImport::ParallelScheduling) + + def collection_method + :issues + end + end + end + + let(:project) { double(:project, id: 4, import_source: 'foo/bar') } + let(:client) { double(:client) } + + describe '#parallel?' do + it 'returns true when running in parallel mode' do + expect(importer_class.new(project, client)).to be_parallel + end + + it 'returns false when running in sequential mode' do + importer = importer_class.new(project, client, parallel: false) + + expect(importer).not_to be_parallel + end + end + + describe '#execute' do + it 'imports data in parallel when running in parallel mode' do + importer = importer_class.new(project, client) + waiter = double(:waiter) + + expect(importer) + .to receive(:parallel_import) + .and_return(waiter) + + expect(importer.execute) + .to eq(waiter) + end + + it 'imports data in parallel when running in sequential mode' do + importer = importer_class.new(project, client, parallel: false) + + expect(importer) + .to receive(:sequential_import) + .and_return([]) + + expect(importer.execute) + .to eq([]) + end + + it 'expires the cache used for tracking already imported objects' do + importer = importer_class.new(project, client) + + expect(importer).to receive(:parallel_import) + + expect(Gitlab::GithubImport::Caching) + .to receive(:expire) + .with(importer.already_imported_cache_key, a_kind_of(Numeric)) + + importer.execute + end + end + + describe '#sequential_import' do + let(:importer) { importer_class.new(project, client, parallel: false) } + + it 'imports data in sequence' do + repr_class = double(:representation_class) + repr_instance = double(:representation_instance) + gh_importer = double(:github_importer) + gh_importer_instance = double(:github_importer_instance) + object = double(:object) + + expect(importer) + .to receive(:each_object_to_import) + .and_yield(object) + + expect(importer) + .to receive(:representation_class) + .and_return(repr_class) + + expect(repr_class) + .to receive(:from_api_response) + .with(object) + .and_return(repr_instance) + + expect(importer) + .to receive(:importer_class) + .and_return(gh_importer) + + expect(gh_importer) + .to receive(:new) + .with(repr_instance, project, client) + .and_return(gh_importer_instance) + + expect(gh_importer_instance) + .to receive(:execute) + + importer.sequential_import + end + end + + describe '#parallel_import' do + let(:importer) { importer_class.new(project, client) } + + it 'imports data in parallel' do + repr_class = double(:representation) + worker_class = double(:worker) + object = double(:object) + + expect(importer) + .to receive(:each_object_to_import) + .and_yield(object) + + expect(importer) + .to receive(:representation_class) + .and_return(repr_class) + + expect(importer) + .to receive(:sidekiq_worker_class) + .and_return(worker_class) + + expect(repr_class) + .to receive(:from_api_response) + .with(object) + .and_return({ title: 'Foo' }) + + expect(worker_class) + .to receive(:perform_async) + .with(project.id, { title: 'Foo' }, an_instance_of(String)) + + expect(importer.parallel_import) + .to be_an_instance_of(Gitlab::JobWaiter) + end + end + + describe '#each_object_to_import' do + let(:importer) { importer_class.new(project, client) } + let(:object) { double(:object) } + + before do + expect(importer) + .to receive(:collection_options) + .and_return({ state: 'all' }) + end + + it 'yields every object to import' do + page = double(:page, objects: [object], number: 1) + + expect(client) + .to receive(:each_page) + .with(:issues, 'foo/bar', { state: 'all', page: 1 }) + .and_yield(page) + + expect(importer.page_counter) + .to receive(:set) + .with(1) + .and_return(true) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) + + expect(importer) + .to receive(:mark_as_imported) + .with(object) + + expect { |b| importer.each_object_to_import(&b) } + .to yield_with_args(object) + end + + it 'resumes from the last page' do + page = double(:page, objects: [object], number: 2) + + expect(importer.page_counter) + .to receive(:current) + .and_return(2) + + expect(client) + .to receive(:each_page) + .with(:issues, 'foo/bar', { state: 'all', page: 2 }) + .and_yield(page) + + expect(importer.page_counter) + .to receive(:set) + .with(2) + .and_return(true) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(false) + + expect(importer) + .to receive(:mark_as_imported) + .with(object) + + expect { |b| importer.each_object_to_import(&b) } + .to yield_with_args(object) + end + + it 'does not yield any objects if the page number was not set' do + page = double(:page, objects: [object], number: 1) + + expect(client) + .to receive(:each_page) + .with(:issues, 'foo/bar', { state: 'all', page: 1 }) + .and_yield(page) + + expect(importer.page_counter) + .to receive(:set) + .with(1) + .and_return(false) + + expect { |b| importer.each_object_to_import(&b) } + .not_to yield_control + end + + it 'does not yield the object if it was already imported' do + page = double(:page, objects: [object], number: 1) + + expect(client) + .to receive(:each_page) + .with(:issues, 'foo/bar', { state: 'all', page: 1 }) + .and_yield(page) + + expect(importer.page_counter) + .to receive(:set) + .with(1) + .and_return(true) + + expect(importer) + .to receive(:already_imported?) + .with(object) + .and_return(true) + + expect(importer) + .not_to receive(:mark_as_imported) + + expect { |b| importer.each_object_to_import(&b) } + .not_to yield_control + end + end + + describe '#already_imported?', :clean_gitlab_redis_cache do + let(:importer) { importer_class.new(project, client) } + + it 'returns false when an object has not yet been imported' do + object = double(:object, id: 10) + + expect(importer) + .to receive(:id_for_already_imported_cache) + .with(object) + .and_return(object.id) + + expect(importer.already_imported?(object)) + .to eq(false) + end + + it 'returns true when an object has already been imported' do + object = double(:object, id: 10) + + allow(importer) + .to receive(:id_for_already_imported_cache) + .with(object) + .and_return(object.id) + + importer.mark_as_imported(object) + + expect(importer.already_imported?(object)) + .to eq(true) + end + end + + describe '#mark_as_imported', :clean_gitlab_redis_cache do + it 'marks an object as already imported' do + object = double(:object, id: 10) + importer = importer_class.new(project, client) + + expect(importer) + .to receive(:id_for_already_imported_cache) + .with(object) + .and_return(object.id) + + expect(Gitlab::GithubImport::Caching) + .to receive(:set_add) + .with(importer.already_imported_cache_key, object.id) + .and_call_original + + importer.mark_as_imported(object) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/diff_note_spec.rb b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb new file mode 100644 index 00000000000..7b0a1ea4948 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/diff_note_spec.rb @@ -0,0 +1,164 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::DiffNote do + let(:hunk) do + '@@ -1 +1 @@ + -Hello + +Hello world' + end + + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + shared_examples 'a DiffNote' do + it 'returns an instance of DiffNote' do + expect(note).to be_an_instance_of(described_class) + end + + context 'the returned DiffNote' do + it 'includes the number of the note' do + expect(note.noteable_id).to eq(42) + end + + it 'includes the file path of the diff' do + expect(note.file_path).to eq('README.md') + end + + it 'includes the commit ID' do + expect(note.commit_id).to eq('123abc') + end + + it 'includes the user details' do + expect(note.author) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(note.author.id).to eq(4) + expect(note.author.login).to eq('alice') + end + + it 'includes the note body' do + expect(note.note).to eq('Hello world') + end + + it 'includes the created timestamp' do + expect(note.created_at).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(note.updated_at).to eq(updated_at) + end + + it 'includes the GitHub ID' do + expect(note.github_id).to eq(1) + end + + it 'returns the noteable type' do + expect(note.noteable_type).to eq('MergeRequest') + end + end + end + + describe '.from_api_response' do + let(:response) do + double( + :response, + html_url: 'https://github.com/foo/bar/pull/42', + path: 'README.md', + commit_id: '123abc', + diff_hunk: hunk, + user: double(:user, id: 4, login: 'alice'), + body: 'Hello world', + created_at: created_at, + updated_at: updated_at, + id: 1 + ) + end + + it_behaves_like 'a DiffNote' do + let(:note) { described_class.from_api_response(response) } + end + + it 'does not set the user if the response did not include a user' do + allow(response) + .to receive(:user) + .and_return(nil) + + note = described_class.from_api_response(response) + + expect(note.author).to be_nil + end + end + + describe '.from_json_hash' do + it_behaves_like 'a DiffNote' do + let(:hash) do + { + 'noteable_type' => 'MergeRequest', + 'noteable_id' => 42, + 'file_path' => 'README.md', + 'commit_id' => '123abc', + 'diff_hunk' => hunk, + 'author' => { 'id' => 4, 'login' => 'alice' }, + 'note' => 'Hello world', + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'github_id' => 1 + } + end + + let(:note) { described_class.from_json_hash(hash) } + end + + it 'does not convert the author if it was not specified' do + hash = { + 'noteable_type' => 'MergeRequest', + 'noteable_id' => 42, + 'file_path' => 'README.md', + 'commit_id' => '123abc', + 'diff_hunk' => hunk, + 'note' => 'Hello world', + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'github_id' => 1 + } + + note = described_class.from_json_hash(hash) + + expect(note.author).to be_nil + end + end + + describe '#line_code' do + it 'returns a String' do + note = described_class.new(diff_hunk: hunk, file_path: 'README.md') + + expect(note.line_code).to be_an_instance_of(String) + end + end + + describe '#diff_hash' do + it 'returns a Hash containing the diff details' do + note = described_class.from_json_hash( + 'noteable_type' => 'MergeRequest', + 'noteable_id' => 42, + 'file_path' => 'README.md', + 'commit_id' => '123abc', + 'diff_hunk' => hunk, + 'author' => { 'id' => 4, 'login' => 'alice' }, + 'note' => 'Hello world', + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'github_id' => 1 + ) + + expect(note.diff_hash).to eq( + diff: hunk, + new_path: 'README.md', + old_path: 'README.md', + a_mode: '100644', + b_mode: '100644', + new_file: false + ) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/expose_attribute_spec.rb b/spec/lib/gitlab/github_import/representation/expose_attribute_spec.rb new file mode 100644 index 00000000000..15de0fe49ff --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/expose_attribute_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::ExposeAttribute do + it 'defines a getter method that returns an attribute value' do + klass = Class.new do + include Gitlab::GithubImport::Representation::ExposeAttribute + + expose_attribute :number + + attr_reader :attributes + + def initialize + @attributes = { number: 42 } + end + end + + expect(klass.new.number).to eq(42) + end +end diff --git a/spec/lib/gitlab/github_import/representation/issue_spec.rb b/spec/lib/gitlab/github_import/representation/issue_spec.rb new file mode 100644 index 00000000000..99330ce42cb --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/issue_spec.rb @@ -0,0 +1,182 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::Issue do + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + shared_examples 'an Issue' do + it 'returns an instance of Issue' do + expect(issue).to be_an_instance_of(described_class) + end + + context 'the returned Issue' do + it 'includes the issue number' do + expect(issue.iid).to eq(42) + end + + it 'includes the issue title' do + expect(issue.title).to eq('My Issue') + end + + it 'includes the issue description' do + expect(issue.description).to eq('This is my issue') + end + + it 'includes the milestone number' do + expect(issue.milestone_number).to eq(4) + end + + it 'includes the issue state' do + expect(issue.state).to eq(:opened) + end + + it 'includes the issue assignees' do + expect(issue.assignees[0]) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(issue.assignees[0].id).to eq(4) + expect(issue.assignees[0].login).to eq('alice') + end + + it 'includes the label names' do + expect(issue.label_names).to eq(%w[bug]) + end + + it 'includes the author details' do + expect(issue.author) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(issue.author.id).to eq(4) + expect(issue.author.login).to eq('alice') + end + + it 'includes the created timestamp' do + expect(issue.created_at).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(issue.updated_at).to eq(updated_at) + end + + it 'is not a pull request' do + expect(issue.pull_request?).to eq(false) + end + end + end + + describe '.from_api_response' do + let(:response) do + double( + :response, + number: 42, + title: 'My Issue', + body: 'This is my issue', + milestone: double(:milestone, number: 4), + state: 'open', + assignees: [double(:user, id: 4, login: 'alice')], + labels: [double(:label, name: 'bug')], + user: double(:user, id: 4, login: 'alice'), + created_at: created_at, + updated_at: updated_at, + pull_request: false + ) + end + + it_behaves_like 'an Issue' do + let(:issue) { described_class.from_api_response(response) } + end + + it 'does not set the user if the response did not include a user' do + allow(response) + .to receive(:user) + .and_return(nil) + + issue = described_class.from_api_response(response) + + expect(issue.author).to be_nil + end + end + + describe '.from_json_hash' do + it_behaves_like 'an Issue' do + let(:hash) do + { + 'iid' => 42, + 'title' => 'My Issue', + 'description' => 'This is my issue', + 'milestone_number' => 4, + 'state' => 'opened', + 'assignees' => [{ 'id' => 4, 'login' => 'alice' }], + 'label_names' => %w[bug], + 'author' => { 'id' => 4, 'login' => 'alice' }, + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'pull_request' => false + } + end + + let(:issue) { described_class.from_json_hash(hash) } + end + + it 'does not convert the author if it was not specified' do + hash = { + 'iid' => 42, + 'title' => 'My Issue', + 'description' => 'This is my issue', + 'milestone_number' => 4, + 'state' => 'opened', + 'assignees' => [{ 'id' => 4, 'login' => 'alice' }], + 'label_names' => %w[bug], + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'pull_request' => false + } + + issue = described_class.from_json_hash(hash) + + expect(issue.author).to be_nil + end + end + + describe '#labels?' do + it 'returns true when the issue has labels assigned' do + issue = described_class.new(label_names: %w[bug]) + + expect(issue.labels?).to eq(true) + end + + it 'returns false when the issue has no labels assigned' do + issue = described_class.new(label_names: []) + + expect(issue.labels?).to eq(false) + end + end + + describe '#pull_request?' do + it 'returns false for an issue' do + issue = described_class.new(pull_request: false) + + expect(issue.pull_request?).to eq(false) + end + + it 'returns true for a pull request' do + issue = described_class.new(pull_request: true) + + expect(issue.pull_request?).to eq(true) + end + end + + describe '#truncated_title' do + it 'truncates the title to 255 characters' do + object = described_class.new(title: 'm' * 300) + + expect(object.truncated_title.length).to eq(255) + end + + it 'does not truncate the title if it is shorter than 255 characters' do + object = described_class.new(title: 'foo') + + expect(object.truncated_title).to eq('foo') + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/note_spec.rb b/spec/lib/gitlab/github_import/representation/note_spec.rb new file mode 100644 index 00000000000..f2c1c66b357 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/note_spec.rb @@ -0,0 +1,107 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::Note do + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + + shared_examples 'a Note' do + it 'returns an instance of Note' do + expect(note).to be_an_instance_of(described_class) + end + + context 'the returned Note' do + it 'includes the noteable ID' do + expect(note.noteable_id).to eq(42) + end + + it 'includes the noteable type' do + expect(note.noteable_type).to eq('Issue') + end + + it 'includes the author details' do + expect(note.author) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(note.author.id).to eq(4) + expect(note.author.login).to eq('alice') + end + + it 'includes the note body' do + expect(note.note).to eq('Hello world') + end + + it 'includes the created timestamp' do + expect(note.created_at).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(note.updated_at).to eq(updated_at) + end + + it 'includes the GitHub ID' do + expect(note.github_id).to eq(1) + end + end + end + + describe '.from_api_response' do + let(:response) do + double( + :response, + html_url: 'https://github.com/foo/bar/issues/42', + user: double(:user, id: 4, login: 'alice'), + body: 'Hello world', + created_at: created_at, + updated_at: updated_at, + id: 1 + ) + end + + it_behaves_like 'a Note' do + let(:note) { described_class.from_api_response(response) } + end + + it 'does not set the user if the response did not include a user' do + allow(response) + .to receive(:user) + .and_return(nil) + + note = described_class.from_api_response(response) + + expect(note.author).to be_nil + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Note' do + let(:hash) do + { + 'noteable_id' => 42, + 'noteable_type' => 'Issue', + 'author' => { 'id' => 4, 'login' => 'alice' }, + 'note' => 'Hello world', + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'github_id' => 1 + } + end + + let(:note) { described_class.from_json_hash(hash) } + end + + it 'does not convert the author if it was not specified' do + hash = { + 'noteable_id' => 42, + 'noteable_type' => 'Issue', + 'note' => 'Hello world', + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'github_id' => 1 + } + + note = described_class.from_json_hash(hash) + + expect(note.author).to be_nil + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/pull_request_spec.rb b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb new file mode 100644 index 00000000000..33f6ff0ae6a --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/pull_request_spec.rb @@ -0,0 +1,288 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::PullRequest do + let(:created_at) { Time.new(2017, 1, 1, 12, 00) } + let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } + let(:merged_at) { Time.new(2017, 1, 1, 12, 17) } + + shared_examples 'a PullRequest' do + it 'returns an instance of PullRequest' do + expect(pr).to be_an_instance_of(described_class) + end + + context 'the returned PullRequest' do + it 'includes the pull request number' do + expect(pr.iid).to eq(42) + end + + it 'includes the pull request title' do + expect(pr.title).to eq('My Pull Request') + end + + it 'includes the pull request description' do + expect(pr.description).to eq('This is my pull request') + end + + it 'includes the source branch name' do + expect(pr.source_branch).to eq('my-feature') + end + + it 'includes the source branch SHA' do + expect(pr.source_branch_sha).to eq('123abc') + end + + it 'includes the target branch name' do + expect(pr.target_branch).to eq('master') + end + + it 'includes the target branch SHA' do + expect(pr.target_branch_sha).to eq('456def') + end + + it 'includes the milestone number' do + expect(pr.milestone_number).to eq(4) + end + + it 'includes the user details' do + expect(pr.author) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(pr.author.id).to eq(4) + expect(pr.author.login).to eq('alice') + end + + it 'includes the assignee details' do + expect(pr.assignee) + .to be_an_instance_of(Gitlab::GithubImport::Representation::User) + + expect(pr.assignee.id).to eq(4) + expect(pr.assignee.login).to eq('alice') + end + + it 'includes the created timestamp' do + expect(pr.created_at).to eq(created_at) + end + + it 'includes the updated timestamp' do + expect(pr.updated_at).to eq(updated_at) + end + + it 'includes the merged timestamp' do + expect(pr.merged_at).to eq(merged_at) + end + + it 'includes the source repository ID' do + expect(pr.source_repository_id).to eq(400) + end + + it 'includes the target repository ID' do + expect(pr.target_repository_id).to eq(200) + end + + it 'includes the source repository owner name' do + expect(pr.source_repository_owner).to eq('alice') + end + + it 'includes the pull request state' do + expect(pr.state).to eq(:merged) + end + end + end + + describe '.from_api_response' do + let(:response) do + double( + :response, + number: 42, + title: 'My Pull Request', + body: 'This is my pull request', + state: 'closed', + head: double( + :head, + sha: '123abc', + ref: 'my-feature', + repo: double(:repo, id: 400), + user: double(:user, id: 4, login: 'alice') + ), + base: double( + :base, + sha: '456def', + ref: 'master', + repo: double(:repo, id: 200) + ), + milestone: double(:milestone, number: 4), + user: double(:user, id: 4, login: 'alice'), + assignee: double(:user, id: 4, login: 'alice'), + created_at: created_at, + updated_at: updated_at, + merged_at: merged_at + ) + end + + it_behaves_like 'a PullRequest' do + let(:pr) { described_class.from_api_response(response) } + end + + it 'does not set the user if the response did not include a user' do + allow(response) + .to receive(:user) + .and_return(nil) + + pr = described_class.from_api_response(response) + + expect(pr.author).to be_nil + end + end + + describe '.from_json_hash' do + it_behaves_like 'a PullRequest' do + let(:hash) do + { + 'iid' => 42, + 'title' => 'My Pull Request', + 'description' => 'This is my pull request', + 'source_branch' => 'my-feature', + 'source_branch_sha' => '123abc', + 'target_branch' => 'master', + 'target_branch_sha' => '456def', + 'source_repository_id' => 400, + 'target_repository_id' => 200, + 'source_repository_owner' => 'alice', + 'state' => 'closed', + 'milestone_number' => 4, + 'author' => { 'id' => 4, 'login' => 'alice' }, + 'assignee' => { 'id' => 4, 'login' => 'alice' }, + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'merged_at' => merged_at.to_s + } + end + + let(:pr) { described_class.from_json_hash(hash) } + end + + it 'does not convert the author if it was not specified' do + hash = { + 'iid' => 42, + 'title' => 'My Pull Request', + 'description' => 'This is my pull request', + 'source_branch' => 'my-feature', + 'source_branch_sha' => '123abc', + 'target_branch' => 'master', + 'target_branch_sha' => '456def', + 'source_repository_id' => 400, + 'target_repository_id' => 200, + 'source_repository_owner' => 'alice', + 'state' => 'closed', + 'milestone_number' => 4, + 'assignee' => { 'id' => 4, 'login' => 'alice' }, + 'created_at' => created_at.to_s, + 'updated_at' => updated_at.to_s, + 'merged_at' => merged_at.to_s + } + + pr = described_class.from_json_hash(hash) + + expect(pr.author).to be_nil + end + end + + describe '#state' do + it 'returns :opened for an open pull request' do + pr = described_class.new(state: :opened) + + expect(pr.state).to eq(:opened) + end + + it 'returns :closed for a closed pull request' do + pr = described_class.new(state: :closed) + + expect(pr.state).to eq(:closed) + end + + it 'returns :merged for a merged pull request' do + pr = described_class.new(state: :closed, merged_at: merged_at) + + expect(pr.state).to eq(:merged) + end + end + + describe '#cross_project?' do + it 'returns false for a pull request submitted from the target project' do + pr = described_class.new(source_repository_id: 1, target_repository_id: 1) + + expect(pr).not_to be_cross_project + end + + it 'returns true for a pull request submitted from a different project' do + pr = described_class.new(source_repository_id: 1, target_repository_id: 2) + + expect(pr).to be_cross_project + end + + it 'returns true if no source repository is present' do + pr = described_class.new(target_repository_id: 2) + + expect(pr).to be_cross_project + end + end + + describe '#formatted_source_branch' do + context 'for a cross-project pull request' do + it 'includes the owner name in the branch name' do + pr = described_class.new( + source_repository_owner: 'foo', + source_branch: 'branch', + target_branch: 'master', + source_repository_id: 1, + target_repository_id: 2 + ) + + expect(pr.formatted_source_branch).to eq('foo:branch') + end + end + + context 'for a regular pull request' do + it 'returns the source branch name' do + pr = described_class.new( + source_repository_owner: 'foo', + source_branch: 'branch', + target_branch: 'master', + source_repository_id: 1, + target_repository_id: 1 + ) + + expect(pr.formatted_source_branch).to eq('branch') + end + end + + context 'for a pull request with the same source and target branches' do + it 'returns a generated source branch name' do + pr = described_class.new( + iid: 1, + source_repository_owner: 'foo', + source_branch: 'branch', + target_branch: 'branch', + source_repository_id: 1, + target_repository_id: 1 + ) + + expect(pr.formatted_source_branch).to eq('branch-1') + end + end + end + + describe '#truncated_title' do + it 'truncates the title to 255 characters' do + object = described_class.new(title: 'm' * 300) + + expect(object.truncated_title.length).to eq(255) + end + + it 'does not truncate the title if it is shorter than 255 characters' do + object = described_class.new(title: 'foo') + + expect(object.truncated_title).to eq('foo') + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/to_hash_spec.rb b/spec/lib/gitlab/github_import/representation/to_hash_spec.rb new file mode 100644 index 00000000000..c296aa0a45b --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/to_hash_spec.rb @@ -0,0 +1,37 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::ToHash do + describe '#to_hash' do + let(:user) { double(:user, attributes: { login: 'alice' }) } + + let(:issue) do + double( + :issue, + attributes: { user: user, assignees: [user], number: 42 } + ) + end + + let(:issue_hash) { issue.to_hash } + + before do + user.extend(described_class) + issue.extend(described_class) + end + + it 'converts an object to a Hash' do + expect(issue_hash).to be_an_instance_of(Hash) + end + + it 'converts nested objects to Hashes' do + expect(issue_hash[:user]).to eq({ login: 'alice' }) + end + + it 'converts Array values to Hashes' do + expect(issue_hash[:assignees]).to eq([{ login: 'alice' }]) + end + + it 'keeps values as-is if they do not respond to #to_hash' do + expect(issue_hash[:number]).to eq(42) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/user_spec.rb b/spec/lib/gitlab/github_import/representation/user_spec.rb new file mode 100644 index 00000000000..4e63e8ea568 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/user_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation::User do + shared_examples 'a User' do + it 'returns an instance of User' do + expect(user).to be_an_instance_of(described_class) + end + + context 'the returned User' do + it 'includes the user ID' do + expect(user.id).to eq(42) + end + + it 'includes the username' do + expect(user.login).to eq('alice') + end + end + end + + describe '.from_api_response' do + it_behaves_like 'a User' do + let(:response) { double(:response, id: 42, login: 'alice') } + let(:user) { described_class.from_api_response(response) } + end + end + + describe '.from_json_hash' do + it_behaves_like 'a User' do + let(:hash) { { 'id' => 42, 'login' => 'alice' } } + let(:user) { described_class.from_json_hash(hash) } + end + end +end diff --git a/spec/lib/gitlab/github_import/representation_spec.rb b/spec/lib/gitlab/github_import/representation_spec.rb new file mode 100644 index 00000000000..0b0610817b0 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Representation do + describe '.symbolize_hash' do + it 'returns a Hash with the keys as Symbols' do + hash = described_class.symbolize_hash('number' => 10) + + expect(hash).to eq({ number: 10 }) + end + + it 'parses timestamp fields into Time instances' do + hash = described_class.symbolize_hash('created_at' => '2017-01-01 12:00') + + expect(hash[:created_at]).to be_an_instance_of(Time) + end + end +end diff --git a/spec/lib/gitlab/github_import/sequential_importer_spec.rb b/spec/lib/gitlab/github_import/sequential_importer_spec.rb new file mode 100644 index 00000000000..6089b0b751f --- /dev/null +++ b/spec/lib/gitlab/github_import/sequential_importer_spec.rb @@ -0,0 +1,37 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::SequentialImporter do + describe '#execute' do + it 'imports a project in sequence' do + repository = double(:repository) + project = double(:project, id: 1, repository: repository) + importer = described_class.new(project, token: 'foo') + + expect_any_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) + .to receive(:execute) + + described_class::SEQUENTIAL_IMPORTERS.each do |klass| + instance = double(:instance) + + expect(klass).to receive(:new) + .with(project, importer.client) + .and_return(instance) + + expect(instance).to receive(:execute) + end + + described_class::PARALLEL_IMPORTERS.each do |klass| + instance = double(:instance) + + expect(klass).to receive(:new) + .with(project, importer.client, parallel: false) + .and_return(instance) + + expect(instance).to receive(:execute) + end + + expect(repository).to receive(:after_import) + expect(importer.execute).to eq(true) + end + end +end diff --git a/spec/lib/gitlab/github_import/user_finder_spec.rb b/spec/lib/gitlab/github_import/user_finder_spec.rb new file mode 100644 index 00000000000..29f4c00d9c7 --- /dev/null +++ b/spec/lib/gitlab/github_import/user_finder_spec.rb @@ -0,0 +1,333 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::UserFinder, :clean_gitlab_redis_cache do + let(:project) { create(:project) } + let(:client) { double(:client) } + let(:finder) { described_class.new(project, client) } + + describe '#author_id_for' do + it 'returns the user ID for the author of an object' do + user = double(:user, id: 4, login: 'kittens') + note = double(:note, author: user) + + expect(finder).to receive(:user_id_for).with(user).and_return(42) + + expect(finder.author_id_for(note)).to eq([42, true]) + end + + it 'returns the ID of the project creator if no user ID could be found' do + user = double(:user, id: 4, login: 'kittens') + note = double(:note, author: user) + + expect(finder).to receive(:user_id_for).with(user).and_return(nil) + + expect(finder.author_id_for(note)).to eq([project.creator_id, false]) + end + + it 'returns the ID of the ghost user when the object has no user' do + note = double(:note, author: nil) + + expect(finder.author_id_for(note)).to eq([User.ghost.id, true]) + end + + it 'returns the ID of the ghost user when the given object is nil' do + expect(finder.author_id_for(nil)).to eq([User.ghost.id, true]) + end + end + + describe '#assignee_id_for' do + it 'returns the user ID for the assignee of an issuable' do + user = double(:user, id: 4, login: 'kittens') + issue = double(:issue, assignee: user) + + expect(finder).to receive(:user_id_for).with(user).and_return(42) + expect(finder.assignee_id_for(issue)).to eq(42) + end + + it 'returns nil if the issuable does not have an assignee' do + issue = double(:issue, assignee: nil) + + expect(finder).not_to receive(:user_id_for) + expect(finder.assignee_id_for(issue)).to be_nil + end + end + + describe '#user_id_for' do + it 'returns the user ID for the given user' do + user = double(:user, id: 4, login: 'kittens') + + expect(finder).to receive(:find).with(user.id, user.login).and_return(42) + expect(finder.user_id_for(user)).to eq(42) + end + end + + describe '#find' do + let(:user) { create(:user) } + + before do + allow(finder).to receive(:email_for_github_username) + .and_return(user.email) + end + + context 'without a cache' do + before do + allow(finder).to receive(:find_from_cache).and_return([false, nil]) + expect(finder).to receive(:find_id_from_database).and_call_original + end + + it 'finds a GitLab user for a GitHub user ID' do + user.identities.create!(provider: :github, extern_uid: 42) + + expect(finder.find(42, user.username)).to eq(user.id) + end + + it 'finds a GitLab user for a GitHub Email address' do + expect(finder.find(42, user.username)).to eq(user.id) + end + end + + context 'with a cache' do + it 'returns the cached user ID' do + expect(finder).to receive(:find_from_cache).and_return([true, user.id]) + expect(finder).not_to receive(:find_id_from_database) + + expect(finder.find(42, user.username)).to eq(user.id) + end + + it 'does not query the database if the cache key exists but is empty' do + expect(finder).to receive(:find_from_cache).and_return([true, nil]) + expect(finder).not_to receive(:find_id_from_database) + + expect(finder.find(42, user.username)).to be_nil + end + end + end + + describe '#find_from_cache' do + it 'retrieves a GitLab user ID for a GitHub user ID' do + expect(finder) + .to receive(:cached_id_for_github_id) + .with(42) + .and_return([true, 4]) + + expect(finder.find_from_cache(42)).to eq([true, 4]) + end + + it 'retrieves a GitLab user ID for a GitHub Email address' do + email = 'kittens@example.com' + + expect(finder) + .to receive(:cached_id_for_github_id) + .with(42) + .and_return([false, nil]) + + expect(finder) + .to receive(:cached_id_for_github_email) + .with(email) + .and_return([true, 4]) + + expect(finder.find_from_cache(42, email)).to eq([true, 4]) + end + + it 'does not query the cache for an Email address when none is given' do + expect(finder) + .to receive(:cached_id_for_github_id) + .with(42) + .and_return([false, nil]) + + expect(finder).not_to receive(:cached_id_for_github_id) + + expect(finder.find_from_cache(42)).to eq([false]) + end + end + + describe '#find_id_from_database' do + let(:user) { create(:user) } + + it 'returns the GitLab user ID for a GitHub user ID' do + user.identities.create!(provider: :github, extern_uid: 42) + + expect(finder.find_id_from_database(42, user.email)).to eq(user.id) + end + + it 'returns the GitLab user ID for a GitHub Email address' do + expect(finder.find_id_from_database(42, user.email)).to eq(user.id) + end + end + + describe '#email_for_github_username' do + let(:email) { 'kittens@example.com' } + + context 'when an Email address is cached' do + it 'reads the Email address from the cache' do + expect(Gitlab::GithubImport::Caching) + .to receive(:read) + .and_return(email) + + expect(client).not_to receive(:user) + expect(finder.email_for_github_username('kittens')).to eq(email) + end + end + + context 'when an Email address is not cached' do + let(:user) { double(:user, email: email) } + + it 'retrieves the Email address from the GitHub API' do + expect(client).to receive(:user).with('kittens').and_return(user) + expect(finder.email_for_github_username('kittens')).to eq(email) + end + + it 'caches the Email address when an Email address is available' do + expect(client).to receive(:user).with('kittens').and_return(user) + + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with(an_instance_of(String), email) + + finder.email_for_github_username('kittens') + end + + it 'returns nil if the user does not exist' do + expect(client) + .to receive(:user) + .with('kittens') + .and_return(nil) + + expect(Gitlab::GithubImport::Caching) + .not_to receive(:write) + + expect(finder.email_for_github_username('kittens')).to be_nil + end + end + end + + describe '#cached_id_for_github_id' do + let(:id) { 4 } + + it 'reads a user ID from the cache' do + Gitlab::GithubImport::Caching + .write(described_class::ID_CACHE_KEY % id, 4) + + expect(finder.cached_id_for_github_id(id)).to eq([true, 4]) + end + + it 'reads a non existing cache key' do + expect(finder.cached_id_for_github_id(id)).to eq([false, nil]) + end + end + + describe '#cached_id_for_github_email' do + let(:email) { 'kittens@example.com' } + + it 'reads a user ID from the cache' do + Gitlab::GithubImport::Caching + .write(described_class::ID_FOR_EMAIL_CACHE_KEY % email, 4) + + expect(finder.cached_id_for_github_email(email)).to eq([true, 4]) + end + + it 'reads a non existing cache key' do + expect(finder.cached_id_for_github_email(email)).to eq([false, nil]) + end + end + + describe '#id_for_github_id' do + let(:id) { 4 } + + it 'queries and caches the user ID for a given GitHub ID' do + expect(finder).to receive(:query_id_for_github_id) + .with(id) + .and_return(42) + + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with(described_class::ID_CACHE_KEY % id, 42) + + finder.id_for_github_id(id) + end + + it 'caches a nil value if no ID could be found' do + expect(finder).to receive(:query_id_for_github_id) + .with(id) + .and_return(nil) + + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with(described_class::ID_CACHE_KEY % id, nil) + + finder.id_for_github_id(id) + end + end + + describe '#id_for_github_email' do + let(:email) { 'kittens@example.com' } + + it 'queries and caches the user ID for a given Email address' do + expect(finder).to receive(:query_id_for_github_email) + .with(email) + .and_return(42) + + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with(described_class::ID_FOR_EMAIL_CACHE_KEY % email, 42) + + finder.id_for_github_email(email) + end + + it 'caches a nil value if no ID could be found' do + expect(finder).to receive(:query_id_for_github_email) + .with(email) + .and_return(nil) + + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .with(described_class::ID_FOR_EMAIL_CACHE_KEY % email, nil) + + finder.id_for_github_email(email) + end + end + + describe '#query_id_for_github_id' do + it 'returns the ID of the user for the given GitHub user ID' do + user = create(:user) + + user.identities.create!(provider: :github, extern_uid: '42') + + expect(finder.query_id_for_github_id(42)).to eq(user.id) + end + + it 'returns nil when no user ID could be found' do + expect(finder.query_id_for_github_id(42)).to be_nil + end + end + + describe '#query_id_for_github_email' do + it 'returns the ID of the user for the given Email address' do + user = create(:user, email: 'kittens@example.com') + + expect(finder.query_id_for_github_email(user.email)).to eq(user.id) + end + + it 'returns nil if no user ID could be found' do + expect(finder.query_id_for_github_email('kittens@example.com')).to be_nil + end + end + + describe '#read_id_from_cache' do + it 'reads an ID from the cache' do + Gitlab::GithubImport::Caching.write('foo', 10) + + expect(finder.read_id_from_cache('foo')).to eq([true, 10]) + end + + it 'reads a cache key with an empty value' do + Gitlab::GithubImport::Caching.write('foo', nil) + + expect(finder.read_id_from_cache('foo')).to eq([true, nil]) + end + + it 'reads a cache key that does not exist' do + expect(finder.read_id_from_cache('foo')).to eq([false, nil]) + end + end +end diff --git a/spec/lib/gitlab/github_import_spec.rb b/spec/lib/gitlab/github_import_spec.rb new file mode 100644 index 00000000000..51414800e8c --- /dev/null +++ b/spec/lib/gitlab/github_import_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +describe Gitlab::GithubImport do + let(:project) { double(:project) } + + describe '.new_client_for' do + it 'returns a new Client with a custom token' do + expect(described_class::Client) + .to receive(:new) + .with('123', parallel: true) + + described_class.new_client_for(project, token: '123') + end + + it 'returns a new Client with a token stored in the import data' do + import_data = double(:import_data, credentials: { user: '123' }) + + expect(project) + .to receive(:import_data) + .and_return(import_data) + + expect(described_class::Client) + .to receive(:new) + .with('123', parallel: true) + + described_class.new_client_for(project) + end + end + + describe '.insert_and_return_id' do + let(:attributes) { { iid: 1, title: 'foo' } } + let(:project) { create(:project) } + + context 'on PostgreSQL' do + it 'returns the ID returned by the query' do + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with(Issue.table_name, [attributes], return_ids: true) + .and_return([10]) + + id = described_class.insert_and_return_id(attributes, project.issues) + + expect(id).to eq(10) + end + end + + context 'on MySQL' do + it 'uses a separate query to retrieve the ID' do + issue = create(:issue, project: project, iid: attributes[:iid]) + + expect(Gitlab::Database) + .to receive(:bulk_insert) + .with(Issue.table_name, [attributes], return_ids: true) + .and_return([]) + + id = described_class.insert_and_return_id(attributes, project.issues) + + expect(id).to eq(issue.id) + end + end + end + + describe '.ghost_user_id', :clean_gitlab_redis_cache do + it 'returns the ID of the ghost user' do + expect(described_class.ghost_user_id).to eq(User.ghost.id) + end + + it 'caches the ghost user ID' do + expect(Gitlab::GithubImport::Caching) + .to receive(:write) + .once + .and_call_original + + 2.times do + described_class.ghost_user_id + end + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 50786ac6324..bf1e97654e5 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -147,19 +147,6 @@ deploy_keys: - user - deploy_keys_projects - projects -cluster: -- cluster_projects -- projects -- user -- provider_gcp -- platform_kubernetes -cluster_projects: -- projects -- clusters -provider_gcp: -- cluster -platform_kubernetes: -- cluster services: - project - service_hook @@ -191,6 +178,7 @@ project: - tags - chat_services - cluster +- clusters - cluster_project - creator - group @@ -300,4 +288,4 @@ push_event_payload: - event issue_assignees: - issue -- assignee
\ No newline at end of file +- assignee diff --git a/spec/lib/gitlab/import_export/merge_request_parser_spec.rb b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb index 473ba40fae7..b793636c4d6 100644 --- a/spec/lib/gitlab/import_export/merge_request_parser_spec.rb +++ b/spec/lib/gitlab/import_export/merge_request_parser_spec.rb @@ -13,7 +13,7 @@ describe Gitlab::ImportExport::MergeRequestParser do let(:parsed_merge_request) do described_class.new(project, - merge_request.diff_head_sha, + 'abcd', merge_request, merge_request.as_json).parse! end @@ -29,4 +29,14 @@ describe Gitlab::ImportExport::MergeRequestParser do it 'has a target branch' do expect(project.repository.branch_exists?(parsed_merge_request.target_branch)).to be true end + + it 'parses a MR that has no source branch' do + allow_any_instance_of(described_class).to receive(:branch_exists?).and_call_original + allow_any_instance_of(described_class).to receive(:branch_exists?).with(merge_request.source_branch).and_return(false) + allow_any_instance_of(described_class).to receive(:fork_merge_request?).and_return(true) + allow(Gitlab::GitalyClient).to receive(:migrate).and_call_original + allow(Gitlab::GitalyClient).to receive(:migrate).with(:fetch_ref).and_return([nil, 0]) + + expect(parsed_merge_request).to eq(merge_request) + end end diff --git a/spec/lib/gitlab/import_sources_spec.rb b/spec/lib/gitlab/import_sources_spec.rb index c5725f47453..f2fa315e3ec 100644 --- a/spec/lib/gitlab/import_sources_spec.rb +++ b/spec/lib/gitlab/import_sources_spec.rb @@ -56,14 +56,14 @@ describe Gitlab::ImportSources do describe '.importer' do import_sources = { - 'github' => Github::Import, + 'github' => Gitlab::GithubImport::ParallelImporter, 'bitbucket' => Gitlab::BitbucketImport::Importer, 'gitlab' => Gitlab::GitlabImport::Importer, 'google_code' => Gitlab::GoogleCodeImport::Importer, 'fogbugz' => Gitlab::FogbugzImport::Importer, 'git' => nil, 'gitlab_project' => Gitlab::ImportExport::Importer, - 'gitea' => Gitlab::GithubImport::Importer + 'gitea' => Gitlab::LegacyGithubImport::Importer } import_sources.each do |name, klass| diff --git a/spec/lib/gitlab/kubernetes/helm_spec.rb b/spec/lib/gitlab/kubernetes/helm_spec.rb new file mode 100644 index 00000000000..15f99b0401f --- /dev/null +++ b/spec/lib/gitlab/kubernetes/helm_spec.rb @@ -0,0 +1,100 @@ +require 'spec_helper' + +describe Gitlab::Kubernetes::Helm do + let(:client) { double('kubernetes client') } + let(:helm) { described_class.new(client) } + let(:namespace) { Gitlab::Kubernetes::Namespace.new(described_class::NAMESPACE, client) } + let(:install_helm) { true } + let(:chart) { 'stable/a_chart' } + let(:application_name) { 'app_name' } + let(:command) { Gitlab::Kubernetes::Helm::InstallCommand.new(application_name, install_helm, chart) } + subject { helm } + + before do + allow(Gitlab::Kubernetes::Namespace).to receive(:new).with(described_class::NAMESPACE, client).and_return(namespace) + end + + describe '#initialize' do + it 'creates a namespace object' do + expect(Gitlab::Kubernetes::Namespace).to receive(:new).with(described_class::NAMESPACE, client) + + subject + end + end + + describe '#install' do + before do + allow(client).to receive(:create_pod).and_return(nil) + allow(namespace).to receive(:ensure_exists!).once + end + + it 'ensures the namespace exists before creating the POD' do + expect(namespace).to receive(:ensure_exists!).once.ordered + expect(client).to receive(:create_pod).once.ordered + + subject.install(command) + end + end + + describe '#installation_status' do + let(:phase) { Gitlab::Kubernetes::Pod::RUNNING } + let(:pod) { Kubeclient::Resource.new(status: { phase: phase }) } # partial representation + + it 'fetches POD phase from kubernetes cluster' do + expect(client).to receive(:get_pod).with(command.pod_name, described_class::NAMESPACE).once.and_return(pod) + + expect(subject.installation_status(command.pod_name)).to eq(phase) + end + end + + describe '#installation_log' do + let(:log) { 'some output' } + let(:response) { RestClient::Response.new(log) } + + it 'fetches POD phase from kubernetes cluster' do + expect(client).to receive(:get_pod_log).with(command.pod_name, described_class::NAMESPACE).once.and_return(response) + + expect(subject.installation_log(command.pod_name)).to eq(log) + end + end + + describe '#delete_installation_pod!' do + it 'deletes the POD from kubernetes cluster' do + expect(client).to receive(:delete_pod).with(command.pod_name, described_class::NAMESPACE).once + + subject.delete_installation_pod!(command.pod_name) + end + end + + describe '#helm_init_command' do + subject { helm.send(:helm_init_command, command) } + + context 'when command.install_helm is true' do + let(:install_helm) { true } + + it { is_expected.to eq('helm init >/dev/null') } + end + + context 'when command.install_helm is false' do + let(:install_helm) { false } + + it { is_expected.to eq('helm init --client-only >/dev/null') } + end + end + + describe '#helm_install_command' do + subject { helm.send(:helm_install_command, command) } + + context 'when command.chart is nil' do + let(:chart) { nil } + + it { is_expected.to be_nil } + end + + context 'when command.chart is set' do + let(:chart) { 'stable/a_chart' } + + it { is_expected.to eq("helm install #{chart} --name #{application_name} --namespace #{namespace.name} >/dev/null")} + end + end +end diff --git a/spec/lib/gitlab/kubernetes/namespace_spec.rb b/spec/lib/gitlab/kubernetes/namespace_spec.rb new file mode 100644 index 00000000000..b3c987f9344 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/namespace_spec.rb @@ -0,0 +1,66 @@ +require 'spec_helper' + +describe Gitlab::Kubernetes::Namespace do + let(:name) { 'a_namespace' } + let(:client) { double('kubernetes client') } + subject { described_class.new(name, client) } + + it { expect(subject.name).to eq(name) } + + describe '#exists?' do + context 'when namespace do not exits' do + let(:exception) { ::KubeException.new(404, "namespace #{name} not found", nil) } + + it 'returns false' do + expect(client).to receive(:get_namespace).with(name).once.and_raise(exception) + + expect(subject.exists?).to be_falsey + end + end + + context 'when namespace exits' do + let(:namespace) { ::Kubeclient::Resource.new(kind: 'Namespace', metadata: { name: name }) } # partial representation + + it 'returns true' do + expect(client).to receive(:get_namespace).with(name).once.and_return(namespace) + + expect(subject.exists?).to be_truthy + end + end + + context 'when cluster cannot be reached' do + let(:exception) { Errno::ECONNREFUSED.new } + + it 'raises exception' do + expect(client).to receive(:get_namespace).with(name).once.and_raise(exception) + + expect { subject.exists? }.to raise_error(exception) + end + end + end + + describe '#create!' do + it 'creates a namespace' do + matcher = have_attributes(metadata: have_attributes(name: name)) + expect(client).to receive(:create_namespace).with(matcher).once + + expect { subject.create! }.not_to raise_error + end + end + + describe '#ensure_exists!' do + it 'checks for existing namespace before creating' do + expect(subject).to receive(:exists?).once.ordered.and_return(false) + expect(subject).to receive(:create!).once.ordered + + subject.ensure_exists! + end + + it 'do not re-create an existing namespace' do + expect(subject).to receive(:exists?).once.and_return(true) + expect(subject).not_to receive(:create!) + + subject.ensure_exists! + end + end +end diff --git a/spec/lib/gitlab/github_import/branch_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb index 426b43f8b51..48655851140 100644 --- a/spec/lib/gitlab/github_import/branch_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/branch_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::BranchFormatter do +describe Gitlab::LegacyGithubImport::BranchFormatter do let(:project) { create(:project, :repository) } let(:commit) { create(:commit, project: project) } let(:repo) { double } diff --git a/spec/lib/gitlab/legacy_github_import/client_spec.rb b/spec/lib/gitlab/legacy_github_import/client_spec.rb new file mode 100644 index 00000000000..80b767abce0 --- /dev/null +++ b/spec/lib/gitlab/legacy_github_import/client_spec.rb @@ -0,0 +1,97 @@ +require 'spec_helper' + +describe Gitlab::LegacyGithubImport::Client do + let(:token) { '123456' } + let(:github_provider) { Settingslogic.new('app_id' => 'asd123', 'app_secret' => 'asd123', 'name' => 'github', 'args' => { 'client_options' => {} }) } + + subject(:client) { described_class.new(token) } + + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([github_provider]) + end + + it 'convert OAuth2 client options to symbols' do + client.client.options.keys.each do |key| + expect(key).to be_kind_of(Symbol) + end + end + + it 'does not crash (e.g. Settingslogic::MissingSetting) when verify_ssl config is not present' do + expect { client.api }.not_to raise_error + end + + context 'when config is missing' do + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([]) + end + + it 'is still possible to get an Octokit client' do + expect { client.api }.not_to raise_error + end + + it 'is not be possible to get an OAuth2 client' do + expect { client.client }.to raise_error(Projects::ImportService::Error) + end + end + + context 'allow SSL verification to be configurable on API' do + before do + github_provider['verify_ssl'] = false + end + + it 'uses supplied value' do + expect(client.client.options[:connection_opts][:ssl]).to eq({ verify: false }) + expect(client.api.connection_options[:ssl]).to eq({ verify: false }) + end + end + + describe '#api_endpoint' do + context 'when provider does not specity an API endpoint' do + it 'uses GitHub root API endpoint' do + expect(client.api.api_endpoint).to eq 'https://api.github.com/' + end + end + + context 'when provider specify a custom API endpoint' do + before do + github_provider['args']['client_options']['site'] = 'https://github.company.com/' + end + + it 'uses the custom API endpoint' do + expect(OmniAuth::Strategies::GitHub).not_to receive(:default_options) + expect(client.api.api_endpoint).to eq 'https://github.company.com/' + end + end + + context 'when given a host' do + subject(:client) { described_class.new(token, host: 'https://try.gitea.io/') } + + it 'builds a endpoint with the given host and the default API version' do + expect(client.api.api_endpoint).to eq 'https://try.gitea.io/api/v3/' + end + end + + context 'when given an API version' do + subject(:client) { described_class.new(token, api_version: 'v3') } + + it 'does not use the API version without a host' do + expect(client.api.api_endpoint).to eq 'https://api.github.com/' + end + end + + context 'when given a host and version' do + subject(:client) { described_class.new(token, host: 'https://try.gitea.io/', api_version: 'v3') } + + it 'builds a endpoint with the given options' do + expect(client.api.api_endpoint).to eq 'https://try.gitea.io/api/v3/' + end + end + end + + it 'does not raise error when rate limit is disabled' do + stub_request(:get, /api.github.com/) + allow(client.api).to receive(:rate_limit!).and_raise(Octokit::NotFound) + + expect { client.issues {} }.not_to raise_error + end +end diff --git a/spec/lib/gitlab/github_import/comment_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb index 035ac8c7c1f..413654e108c 100644 --- a/spec/lib/gitlab/github_import/comment_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::CommentFormatter do +describe Gitlab::LegacyGithubImport::CommentFormatter do let(:client) { double } let(:project) { create(:project) } let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } diff --git a/spec/lib/gitlab/github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index d570f34985b..20514486727 100644 --- a/spec/lib/gitlab/github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' -describe Gitlab::GithubImport::Importer do - shared_examples 'Gitlab::GithubImport::Importer#execute' do +describe Gitlab::LegacyGithubImport::Importer do + shared_examples 'Gitlab::LegacyGithubImport::Importer#execute' do let(:expected_not_called) { [] } before do @@ -35,7 +35,7 @@ describe Gitlab::GithubImport::Importer do end end - shared_examples 'Gitlab::GithubImport::Importer#execute an error occurs' do + shared_examples 'Gitlab::LegacyGithubImport::Importer#execute an error occurs' do before do allow(project).to receive(:import_data).and_return(double.as_null_object) @@ -178,7 +178,7 @@ describe Gitlab::GithubImport::Importer do end end - shared_examples 'Gitlab::GithubImport unit-testing' do + shared_examples 'Gitlab::LegacyGithubImport unit-testing' do describe '#clean_up_restored_branches' do subject { described_class.new(project) } @@ -188,7 +188,7 @@ describe Gitlab::GithubImport::Importer do end context 'when pull request stills open' do - let(:gh_pull_request) { Gitlab::GithubImport::PullRequestFormatter.new(project, pull_request) } + let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, pull_request) } it 'does not remove branches' do expect(subject).not_to receive(:remove_branch) @@ -197,7 +197,7 @@ describe Gitlab::GithubImport::Importer do end context 'when pull request is closed' do - let(:gh_pull_request) { Gitlab::GithubImport::PullRequestFormatter.new(project, closed_pull_request) } + let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, closed_pull_request) } it 'does remove branches' do expect(subject).to receive(:remove_branch).at_least(2).times @@ -262,14 +262,14 @@ describe Gitlab::GithubImport::Importer do let(:repo_root) { 'https://github.com' } subject { described_class.new(project) } - it_behaves_like 'Gitlab::GithubImport::Importer#execute' - it_behaves_like 'Gitlab::GithubImport::Importer#execute an error occurs' - it_behaves_like 'Gitlab::GithubImport unit-testing' + it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute' + it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute an error occurs' + it_behaves_like 'Gitlab::LegacyGithubImport unit-testing' describe '#client' do it 'instantiates a Client' do allow(project).to receive(:import_data).and_return(double(credentials: credentials)) - expect(Gitlab::GithubImport::Client).to receive(:new).with( + expect(Gitlab::LegacyGithubImport::Client).to receive(:new).with( credentials[:user], {} ) @@ -288,16 +288,16 @@ describe Gitlab::GithubImport::Importer do project.update(import_type: 'gitea', import_url: "#{repo_root}/foo/group/project.git") end - it_behaves_like 'Gitlab::GithubImport::Importer#execute' do + it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute' do let(:expected_not_called) { [:import_releases] } end - it_behaves_like 'Gitlab::GithubImport::Importer#execute an error occurs' - it_behaves_like 'Gitlab::GithubImport unit-testing' + it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute an error occurs' + it_behaves_like 'Gitlab::LegacyGithubImport unit-testing' describe '#client' do it 'instantiates a Client' do allow(project).to receive(:import_data).and_return(double(credentials: credentials)) - expect(Gitlab::GithubImport::Client).to receive(:new).with( + expect(Gitlab::LegacyGithubImport::Client).to receive(:new).with( credentials[:user], { host: "#{repo_root}:443/foo", api_version: 'v1' } ) diff --git a/spec/lib/gitlab/github_import/issuable_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb index 05294d227bd..3b5d8945344 100644 --- a/spec/lib/gitlab/github_import/issuable_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::IssuableFormatter do +describe Gitlab::LegacyGithubImport::IssuableFormatter do let(:raw_data) do double(number: 42) end diff --git a/spec/lib/gitlab/github_import/issue_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb index 0fc56d92aa6..1a4d5dbfb70 100644 --- a/spec/lib/gitlab/github_import/issue_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::IssueFormatter do +describe Gitlab::LegacyGithubImport::IssueFormatter do let(:client) { double } let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } @@ -30,7 +30,7 @@ describe Gitlab::GithubImport::IssueFormatter do allow(client).to receive(:user).and_return(octocat) end - shared_examples 'Gitlab::GithubImport::IssueFormatter#attributes' do + shared_examples 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' do context 'when issue is open' do let(:raw_data) { double(base_data.merge(state: 'open')) } @@ -135,7 +135,7 @@ describe Gitlab::GithubImport::IssueFormatter do end end - shared_examples 'Gitlab::GithubImport::IssueFormatter#number' do + shared_examples 'Gitlab::LegacyGithubImport::IssueFormatter#number' do let(:raw_data) { double(base_data.merge(number: 1347)) } it 'returns issue number' do @@ -144,8 +144,8 @@ describe Gitlab::GithubImport::IssueFormatter do end context 'when importing a GitHub project' do - it_behaves_like 'Gitlab::GithubImport::IssueFormatter#attributes' - it_behaves_like 'Gitlab::GithubImport::IssueFormatter#number' + it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#number' end context 'when importing a Gitea project' do @@ -153,8 +153,8 @@ describe Gitlab::GithubImport::IssueFormatter do project.update(import_type: 'gitea') end - it_behaves_like 'Gitlab::GithubImport::IssueFormatter#attributes' - it_behaves_like 'Gitlab::GithubImport::IssueFormatter#number' + it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#number' end describe '#has_comments?' do diff --git a/spec/lib/gitlab/github_import/label_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb index 83fdd2cc415..0d1d04f1bf6 100644 --- a/spec/lib/gitlab/github_import/label_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::LabelFormatter do +describe Gitlab::LegacyGithubImport::LabelFormatter do let(:project) { create(:project) } let(:raw) { double(name: 'improvements', color: 'e6e6e6') } diff --git a/spec/lib/gitlab/github_import/milestone_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb index 683fa51b78e..1db4bbb568c 100644 --- a/spec/lib/gitlab/github_import/milestone_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::MilestoneFormatter do +describe Gitlab::LegacyGithubImport::MilestoneFormatter do let(:project) { create(:project) } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } @@ -19,7 +19,7 @@ describe Gitlab::GithubImport::MilestoneFormatter do subject(:formatter) { described_class.new(project, raw_data) } - shared_examples 'Gitlab::GithubImport::MilestoneFormatter#attributes' do + shared_examples 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' do let(:data) { base_data.merge(iid_attr => 1347) } context 'when milestone is open' do @@ -82,7 +82,7 @@ describe Gitlab::GithubImport::MilestoneFormatter do end context 'when importing a GitHub project' do - it_behaves_like 'Gitlab::GithubImport::MilestoneFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' end context 'when importing a Gitea project' do @@ -91,6 +91,6 @@ describe Gitlab::GithubImport::MilestoneFormatter do project.update(import_type: 'gitea') end - it_behaves_like 'Gitlab::GithubImport::MilestoneFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' end end diff --git a/spec/lib/gitlab/github_import/project_creator_spec.rb b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb index 948e7469a18..737c9a624e0 100644 --- a/spec/lib/gitlab/github_import/project_creator_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::ProjectCreator do +describe Gitlab::LegacyGithubImport::ProjectCreator do let(:user) { create(:user) } let(:namespace) { create(:group, owner: user) } diff --git a/spec/lib/gitlab/github_import/pull_request_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb index 2e42f6239b7..267a41e3f32 100644 --- a/spec/lib/gitlab/github_import/pull_request_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::PullRequestFormatter do +describe Gitlab::LegacyGithubImport::PullRequestFormatter do let(:client) { double } let(:project) { create(:project, :repository) } let(:source_sha) { create(:commit, project: project).id } @@ -44,7 +44,7 @@ describe Gitlab::GithubImport::PullRequestFormatter do allow(client).to receive(:user).and_return(octocat) end - shared_examples 'Gitlab::GithubImport::PullRequestFormatter#attributes' do + shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' do context 'when pull request is open' do let(:raw_data) { double(base_data.merge(state: 'open')) } @@ -189,7 +189,7 @@ describe Gitlab::GithubImport::PullRequestFormatter do end end - shared_examples 'Gitlab::GithubImport::PullRequestFormatter#number' do + shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' do let(:raw_data) { double(base_data) } it 'returns pull request number' do @@ -197,7 +197,7 @@ describe Gitlab::GithubImport::PullRequestFormatter do end end - shared_examples 'Gitlab::GithubImport::PullRequestFormatter#source_branch_name' do + shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' do context 'when source branch exists' do let(:raw_data) { double(base_data) } @@ -231,7 +231,7 @@ describe Gitlab::GithubImport::PullRequestFormatter do end end - shared_examples 'Gitlab::GithubImport::PullRequestFormatter#target_branch_name' do + shared_examples 'Gitlab::LegacyGithubImport::PullRequestFormatter#target_branch_name' do context 'when target branch exists' do let(:raw_data) { double(base_data) } @@ -250,10 +250,10 @@ describe Gitlab::GithubImport::PullRequestFormatter do end context 'when importing a GitHub project' do - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#attributes' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#number' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#source_branch_name' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#target_branch_name' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#target_branch_name' end context 'when importing a Gitea project' do @@ -261,10 +261,10 @@ describe Gitlab::GithubImport::PullRequestFormatter do project.update(import_type: 'gitea') end - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#attributes' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#number' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#source_branch_name' - it_behaves_like 'Gitlab::GithubImport::PullRequestFormatter#target_branch_name' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#target_branch_name' end describe '#valid?' do diff --git a/spec/lib/gitlab/github_import/release_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb index 926bf725d6a..082e3b36dd0 100644 --- a/spec/lib/gitlab/github_import/release_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::ReleaseFormatter do +describe Gitlab::LegacyGithubImport::ReleaseFormatter do let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } let(:octocat) { double(id: 123456, login: 'octocat') } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } diff --git a/spec/lib/gitlab/github_import/user_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb index 98e3a7c28b9..3cd096eb0ad 100644 --- a/spec/lib/gitlab/github_import/user_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::UserFormatter do +describe Gitlab::LegacyGithubImport::UserFormatter do let(:client) { double } let(:octocat) { double(id: 123456, login: 'octocat', email: 'octocat@example.com') } diff --git a/spec/lib/gitlab/github_import/wiki_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/wiki_formatter_spec.rb index 2662cc20b32..7723533aee2 100644 --- a/spec/lib/gitlab/github_import/wiki_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/wiki_formatter_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::GithubImport::WikiFormatter do +describe Gitlab::LegacyGithubImport::WikiFormatter do let(:project) do create(:project, namespace: create(:namespace, path: 'gitlabhq'), diff --git a/spec/lib/gitlab/metrics/background_transaction_spec.rb b/spec/lib/gitlab/metrics/background_transaction_spec.rb index 96052b8dc2f..17445fe6de5 100644 --- a/spec/lib/gitlab/metrics/background_transaction_spec.rb +++ b/spec/lib/gitlab/metrics/background_transaction_spec.rb @@ -10,4 +10,10 @@ describe Gitlab::Metrics::BackgroundTransaction do expect(subject.action).to eq('TestWorker#perform') end end + + describe '#label' do + it 'returns labels based on class name' do + expect(subject.labels).to eq(controller: 'TestWorker', action: 'perform') + end + end end diff --git a/spec/lib/gitlab/middleware/read_only_spec.rb b/spec/lib/gitlab/middleware/read_only_spec.rb index 86be06ff595..b14735943a5 100644 --- a/spec/lib/gitlab/middleware/read_only_spec.rb +++ b/spec/lib/gitlab/middleware/read_only_spec.rb @@ -91,13 +91,6 @@ describe Gitlab::Middleware::ReadOnly do end context 'whitelisted requests' do - it 'expects DELETE request to logout to be allowed' do - response = request.delete('/users/sign_out') - - expect(response).not_to be_a_redirect - expect(subject).not_to disallow_request - end - it 'expects a POST internal request to be allowed' do response = request.post("/api/#{API::API.version}/internal") diff --git a/spec/lib/gitlab/path_regex_spec.rb b/spec/lib/gitlab/path_regex_spec.rb index ee63c9338c5..0ae90069b7f 100644 --- a/spec/lib/gitlab/path_regex_spec.rb +++ b/spec/lib/gitlab/path_regex_spec.rb @@ -45,21 +45,16 @@ describe Gitlab::PathRegex do Found new routes that could cause conflicts with existing namespaced routes for groups or projects. - Add <#{missing_words.join(', ')}> to `Gitlab::PathRegex::#{constant_name} - to make sure no projects or namespaces can be created with those paths. - - To rename any existing records with those paths you can use the - `Gitlab::Database::RenameReservedpathsMigration::<VERSION>.#{migration_helper}` - migration helper. - - Make sure to make a note of the renamed records in the release blog post. + Nest <#{missing_words.join(', ')}> in a route containing `-`, that way + we know there will be no conflicts with groups or projects created with those + paths. MISSING end if additional_words.any? message += <<-ADDITIONAL - Why are <#{additional_words.join(', ')}> in `#{constant_name}`? + Is <#{additional_words.join(', ')}> in `#{constant_name}` required? If they are really required, update these specs to reflect that. ADDITIONAL @@ -157,16 +152,7 @@ describe Gitlab::PathRegex do let(:paths_after_group_id) do group_routes.map do |route| route.gsub(STARTING_WITH_GROUP, '').split('/').first - end.uniq + ee_paths_after_group_id - end - - let(:ee_paths_after_group_id) do - %w(analytics - ldap - ldap_group_links - notification_setting - audit_events - pipeline_quota hooks) + end.uniq end describe 'TOP_LEVEL_ROUTES' do @@ -225,8 +211,6 @@ describe Gitlab::PathRegex do it 'accepts group routes' do expect(subject).to match('activity/') - expect(subject).to match('group_members/') - expect(subject).to match('labels/') end it 'is not case sensitive' do @@ -258,8 +242,6 @@ describe Gitlab::PathRegex do it 'accepts group routes' do expect(subject).to match('activity/') - expect(subject).to match('group_members/') - expect(subject).to match('labels/') end end @@ -280,8 +262,6 @@ describe Gitlab::PathRegex do it 'accepts group routes' do expect(subject).to match('activity/more/') - expect(subject).to match('group_members/more/') - expect(subject).to match('labels/more/') end end end @@ -303,9 +283,7 @@ describe Gitlab::PathRegex do end it 'rejects group routes' do - expect(subject).not_to match('root/activity/') - expect(subject).not_to match('root/group_members/') - expect(subject).not_to match('root/labels/') + expect(subject).not_to match('root/-/') end end @@ -325,9 +303,7 @@ describe Gitlab::PathRegex do end it 'rejects group routes' do - expect(subject).not_to match('root/activity/more/') - expect(subject).not_to match('root/group_members/more/') - expect(subject).not_to match('root/labels/more/') + expect(subject).not_to match('root/-/') end end end @@ -360,9 +336,7 @@ describe Gitlab::PathRegex do end it 'accepts group routes' do - expect(subject).to match('activity/') - expect(subject).to match('group_members/') - expect(subject).to match('labels/') + expect(subject).to match('analytics/') end it 'is not case sensitive' do @@ -393,9 +367,7 @@ describe Gitlab::PathRegex do end it 'accepts group routes' do - expect(subject).to match('root/activity/') - expect(subject).to match('root/group_members/') - expect(subject).to match('root/labels/') + expect(subject).to match('root/analytics/') end it 'is not case sensitive' do diff --git a/spec/models/clusters/applications/helm_spec.rb b/spec/models/clusters/applications/helm_spec.rb new file mode 100644 index 00000000000..f8855079842 --- /dev/null +++ b/spec/models/clusters/applications/helm_spec.rb @@ -0,0 +1,102 @@ +require 'rails_helper' + +describe Clusters::Applications::Helm do + it { is_expected.to belong_to(:cluster) } + it { is_expected.to validate_presence_of(:cluster) } + + describe '#name' do + it 'is .application_name' do + expect(subject.name).to eq(described_class.application_name) + end + + it 'is recorded in Clusters::Cluster::APPLICATIONS' do + expect(Clusters::Cluster::APPLICATIONS[subject.name]).to eq(described_class) + end + end + + describe '#version' do + it 'defaults to Gitlab::Kubernetes::Helm::HELM_VERSION' do + expect(subject.version).to eq(Gitlab::Kubernetes::Helm::HELM_VERSION) + end + end + + describe '#status' do + let(:cluster) { create(:cluster) } + + subject { described_class.new(cluster: cluster) } + + it 'defaults to :not_installable' do + expect(subject.status_name).to be(:not_installable) + end + + context 'when platform kubernetes is defined' do + let(:cluster) { create(:cluster, :provided_by_gcp) } + + it 'defaults to :installable' do + expect(subject.status_name).to be(:installable) + end + end + end + + describe '#install_command' do + it 'has all the needed information' do + expect(subject.install_command).to have_attributes(name: subject.name, install_helm: true, chart: nil) + end + end + + describe 'status state machine' do + describe '#make_installing' do + subject { create(:cluster_applications_helm, :scheduled) } + + it 'is installing' do + subject.make_installing! + + expect(subject).to be_installing + end + end + + describe '#make_installed' do + subject { create(:cluster_applications_helm, :installing) } + + it 'is installed' do + subject.make_installed + + expect(subject).to be_installed + end + end + + describe '#make_errored' do + subject { create(:cluster_applications_helm, :installing) } + let(:reason) { 'some errors' } + + it 'is errored' do + subject.make_errored(reason) + + expect(subject).to be_errored + expect(subject.status_reason).to eq(reason) + end + end + + describe '#make_scheduled' do + subject { create(:cluster_applications_helm, :installable) } + + it 'is scheduled' do + subject.make_scheduled + + expect(subject).to be_scheduled + end + + describe 'when was errored' do + subject { create(:cluster_applications_helm, :errored) } + + it 'clears #status_reason' do + expect(subject.status_reason).not_to be_nil + + subject.make_scheduled! + + expect(subject.status_reason).to be_nil + end + end + end + end +end diff --git a/spec/models/clusters/applications/ingress_spec.rb b/spec/models/clusters/applications/ingress_spec.rb new file mode 100644 index 00000000000..b83472e1944 --- /dev/null +++ b/spec/models/clusters/applications/ingress_spec.rb @@ -0,0 +1,108 @@ +require 'rails_helper' + +describe Clusters::Applications::Ingress do + it { is_expected.to belong_to(:cluster) } + it { is_expected.to validate_presence_of(:cluster) } + + describe '#name' do + it 'is .application_name' do + expect(subject.name).to eq(described_class.application_name) + end + + it 'is recorded in Clusters::Cluster::APPLICATIONS' do + expect(Clusters::Cluster::APPLICATIONS[subject.name]).to eq(described_class) + end + end + + describe '#status' do + let(:cluster) { create(:cluster, :provided_by_gcp) } + + subject { described_class.new(cluster: cluster) } + + it 'defaults to :not_installable' do + expect(subject.status_name).to be(:not_installable) + end + + context 'when application helm is scheduled' do + before do + create(:cluster_applications_helm, :scheduled, cluster: cluster) + end + + it 'defaults to :not_installable' do + expect(subject.status_name).to be(:not_installable) + end + end + + context 'when application helm is installed' do + before do + create(:cluster_applications_helm, :installed, cluster: cluster) + end + + it 'defaults to :installable' do + expect(subject.status_name).to be(:installable) + end + end + end + + describe '#install_command' do + it 'has all the needed information' do + expect(subject.install_command).to have_attributes(name: subject.name, install_helm: false, chart: subject.chart) + end + end + + describe 'status state machine' do + describe '#make_installing' do + subject { create(:cluster_applications_ingress, :scheduled) } + + it 'is installing' do + subject.make_installing! + + expect(subject).to be_installing + end + end + + describe '#make_installed' do + subject { create(:cluster_applications_ingress, :installing) } + + it 'is installed' do + subject.make_installed + + expect(subject).to be_installed + end + end + + describe '#make_errored' do + subject { create(:cluster_applications_ingress, :installing) } + let(:reason) { 'some errors' } + + it 'is errored' do + subject.make_errored(reason) + + expect(subject).to be_errored + expect(subject.status_reason).to eq(reason) + end + end + + describe '#make_scheduled' do + subject { create(:cluster_applications_ingress, :installable) } + + it 'is scheduled' do + subject.make_scheduled + + expect(subject).to be_scheduled + end + + describe 'when was errored' do + subject { create(:cluster_applications_ingress, :errored) } + + it 'clears #status_reason' do + expect(subject.status_reason).not_to be_nil + + subject.make_scheduled! + + expect(subject.status_reason).to be_nil + end + end + end + end +end diff --git a/spec/models/clusters/cluster_spec.rb b/spec/models/clusters/cluster_spec.rb index 12123a3d753..b91a5e7a272 100644 --- a/spec/models/clusters/cluster_spec.rb +++ b/spec/models/clusters/cluster_spec.rb @@ -178,4 +178,25 @@ describe Clusters::Cluster do it { is_expected.to be_nil } end end + + describe '#applications' do + set(:cluster) { create(:cluster) } + + subject { cluster.applications } + + context 'when none of applications are created' do + it 'returns a list of a new objects' do + is_expected.not_to be_empty + end + end + + context 'when applications are created' do + let!(:helm) { create(:cluster_applications_helm, cluster: cluster) } + let!(:ingress) { create(:cluster_applications_ingress, cluster: cluster) } + + it 'returns a list of created applications' do + is_expected.to contain_exactly(helm, ingress) + end + end + end end diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index c8caa11b8b0..d4052a64570 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -65,12 +65,6 @@ describe Group do expect(group).not_to be_valid end - - it 'rejects reserved group paths' do - group = build(:group, path: 'activity', parent: create(:group)) - - expect(group).not_to be_valid - end end describe '#visibility_level_allowed_by_parent' do diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index 7617e1f89b1..1c629155e1e 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -7,7 +7,7 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do let(:project) { build_stubbed(:kubernetes_project) } let(:service) { project.kubernetes_service } - describe "Associations" do + describe 'Associations' do it { is_expected.to belong_to :project } end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 6185f55c1dc..e684389af5d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3016,4 +3016,77 @@ describe Project do end end end + + describe '#after_import' do + let(:project) { build(:project) } + + it 'runs the correct hooks' do + expect(project.repository).to receive(:after_import) + expect(project).to receive(:import_finish) + expect(project).to receive(:update_project_counter_caches) + expect(project).to receive(:remove_import_jid) + + project.after_import + end + end + + describe '#update_project_counter_caches' do + let(:project) { create(:project) } + + it 'updates all project counter caches' do + expect_any_instance_of(Projects::OpenIssuesCountService) + .to receive(:refresh_cache) + .and_call_original + + expect_any_instance_of(Projects::OpenMergeRequestsCountService) + .to receive(:refresh_cache) + .and_call_original + + project.update_project_counter_caches + end + end + + describe '#remove_import_jid', :clean_gitlab_redis_cache do + let(:project) { } + + context 'without an import JID' do + it 'does nothing' do + project = create(:project) + + expect(Gitlab::SidekiqStatus) + .not_to receive(:unset) + + project.remove_import_jid + end + end + + context 'with an import JID' do + it 'unsets the import JID' do + project = create(:project, import_jid: '123') + + expect(Gitlab::SidekiqStatus) + .to receive(:unset) + .with('123') + .and_call_original + + project.remove_import_jid + + expect(project.import_jid).to be_nil + end + end + end + + describe '#wiki_repository_exists?' do + it 'returns true when the wiki repository exists' do + project = create(:project, :wiki_repo) + + expect(project.wiki_repository_exists?).to eq(true) + end + + it 'returns false when the wiki repository does not exist' do + project = create(:project) + + expect(project.wiki_repository_exists?).to eq(false) + end + end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index d2f97009ad9..a3ebf649aa8 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -841,6 +841,19 @@ describe User do end end + describe '.by_any_email' do + it 'returns an ActiveRecord::Relation' do + expect(described_class.by_any_email('foo@example.com')) + .to be_a_kind_of(ActiveRecord::Relation) + end + + it 'returns a relation of users' do + user = create(:user) + + expect(described_class.by_any_email(user.email)).to eq([user]) + end + end + describe '.search' do let!(:user) { create(:user, name: 'user', username: 'usern', email: 'email@gmail.com') } let!(:user2) { create(:user, name: 'user name', username: 'username', email: 'someemail@gmail.com') } diff --git a/spec/routing/group_routing_spec.rb b/spec/routing/group_routing_spec.rb new file mode 100644 index 00000000000..7a4c8304e62 --- /dev/null +++ b/spec/routing/group_routing_spec.rb @@ -0,0 +1,127 @@ +require 'spec_helper' + +describe "Groups", "routing" do + let(:group_path) { 'complex.group-namegit' } + let!(:group) { create(:group, path: group_path) } + + it "to #show" do + expect(get("/groups/#{group_path}")).to route_to('groups#show', id: group_path) + end + + it "also supports nested groups" do + nested_group = create(:group, parent: group) + expect(get("/#{group_path}/#{nested_group.path}")).to route_to('groups#show', id: "#{group_path}/#{nested_group.path}") + end + + it "also display group#show on the short path" do + expect(get("/#{group_path}")).to route_to('groups#show', id: group_path) + end + + it "to #activity" do + expect(get("/groups/#{group_path}/-/activity")).to route_to('groups#activity', id: group_path) + end + + it "to #issues" do + expect(get("/groups/#{group_path}/-/issues")).to route_to('groups#issues', id: group_path) + end + + it "to #members" do + expect(get("/groups/#{group_path}/-/group_members")).to route_to('groups/group_members#index', group_id: group_path) + end + + it "to #labels" do + expect(get("/groups/#{group_path}/-/labels")).to route_to('groups/labels#index', group_id: group_path) + end + + it "to #milestones" do + expect(get("/groups/#{group_path}/-/milestones")).to route_to('groups/milestones#index', group_id: group_path) + end + + describe 'legacy redirection' do + describe 'labels' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/labels", "/groups/complex.group-namegit/-/labels/" do + let(:resource) { create(:group, parent: group, path: 'labels') } + end + end + + describe 'group_members' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/group_members", "/groups/complex.group-namegit/-/group_members/" do + let(:resource) { create(:group, parent: group, path: 'group_members') } + end + end + + describe 'avatar' do + it 'routes to the avatars controller' do + expect(delete("/groups/#{group_path}/-/avatar")) + .to route_to(group_id: group_path, + controller: 'groups/avatars', + action: 'destroy') + end + end + + describe 'milestones' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/milestones", "/groups/complex.group-namegit/-/milestones/" do + let(:resource) { create(:group, parent: group, path: 'milestones') } + end + + context 'nested routes' do + include RSpec::Rails::RequestExampleGroup + + let(:milestone) { create(:milestone, group: group) } + + it 'redirects the nested routes' do + request = get("/groups/#{group_path}/milestones/#{milestone.id}/merge_requests") + expect(request).to redirect_to("/groups/#{group_path}/-/milestones/#{milestone.id}/merge_requests") + end + end + + context 'with a query string' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/milestones?hello=world", "/groups/complex.group-namegit/-/milestones/?hello=world" do + let(:resource) { create(:group, parent: group, path: 'milestones') } + end + + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/milestones?milestones=/milestones", "/groups/complex.group-namegit/-/milestones/?milestones=/milestones" do + let(:resource) { create(:group, parent: group, path: 'milestones') } + end + end + end + + describe 'edit' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/edit", "/groups/complex.group-namegit/-/edit/" do + let(:resource) do + pending('still rejected because of the wildcard reserved word') + create(:group, parent: group, path: 'edit') + end + end + end + + describe 'issues' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/issues", "/groups/complex.group-namegit/-/issues/" do + let(:resource) { create(:group, parent: group, path: 'issues') } + end + end + + describe 'merge_requests' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/merge_requests", "/groups/complex.group-namegit/-/merge_requests/" do + let(:resource) { create(:group, parent: group, path: 'merge_requests') } + end + end + + describe 'projects' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/projects", "/groups/complex.group-namegit/-/projects/" do + let(:resource) { create(:group, parent: group, path: 'projects') } + end + end + + describe 'activity' do + it_behaves_like 'redirecting a legacy path', "/groups/complex.group-namegit/activity", "/groups/complex.group-namegit/-/activity/" do + let(:resource) { create(:group, parent: group, path: 'activity') } + end + + it_behaves_like 'redirecting a legacy path', "/groups/activity/activity", "/groups/activity/-/activity/" do + let!(:parent) { create(:group, path: 'activity') } + let(:resource) { create(:group, parent: parent, path: 'activity') } + end + end + end +end diff --git a/spec/routing/routing_spec.rb b/spec/routing/routing_spec.rb index 609481603af..91aefa84d0e 100644 --- a/spec/routing/routing_spec.rb +++ b/spec/routing/routing_spec.rb @@ -257,8 +257,10 @@ describe "Authentication", "routing" do expect(post("/users/sign_in")).to route_to('sessions#create') end - it "DELETE /users/sign_out" do - expect(delete("/users/sign_out")).to route_to('sessions#destroy') + # sign_out with GET instead of DELETE facilitates ad-hoc single-sign-out processes + # (https://gitlab.com/gitlab-org/gitlab-ce/issues/39708) + it "GET /users/sign_out" do + expect(get("/users/sign_out")).to route_to('sessions#destroy') end it "POST /users/password" do @@ -278,36 +280,6 @@ describe "Authentication", "routing" do end end -describe "Groups", "routing" do - let(:name) { 'complex.group-namegit' } - let!(:group) { create(:group, name: name) } - - it "to #show" do - expect(get("/groups/#{name}")).to route_to('groups#show', id: name) - end - - it "also supports nested groups" do - nested_group = create(:group, parent: group) - expect(get("/#{name}/#{nested_group.name}")).to route_to('groups#show', id: "#{name}/#{nested_group.name}") - end - - it "also display group#show on the short path" do - expect(get("/#{name}")).to route_to('groups#show', id: name) - end - - it "to #activity" do - expect(get("/groups/#{name}/activity")).to route_to('groups#activity', id: name) - end - - it "to #issues" do - expect(get("/groups/#{name}/issues")).to route_to('groups#issues', id: name) - end - - it "to #members" do - expect(get("/groups/#{name}/group_members")).to route_to('groups/group_members#index', group_id: name) - end -end - describe HealthCheckController, 'routing' do it 'to #index' do expect(get('/health_check')).to route_to('health_check#index') diff --git a/spec/serializers/cluster_application_entity_spec.rb b/spec/serializers/cluster_application_entity_spec.rb new file mode 100644 index 00000000000..87c7b2ad36e --- /dev/null +++ b/spec/serializers/cluster_application_entity_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe ClusterApplicationEntity do + describe '#as_json' do + let(:application) { build(:cluster_applications_helm) } + subject { described_class.new(application).as_json } + + it 'has name' do + expect(subject[:name]).to eq(application.name) + end + + it 'has status' do + expect(subject[:status]).to eq(:not_installable) + end + + it 'has no status_reason' do + expect(subject[:status_reason]).to be_nil + end + + context 'when application is errored' do + let(:application) { build(:cluster_applications_helm, :errored) } + + it 'has corresponded data' do + expect(subject[:status]).to eq(:errored) + expect(subject[:status_reason]).not_to be_nil + expect(subject[:status_reason]).to eq(application.status_reason) + end + end + end +end diff --git a/spec/serializers/cluster_entity_spec.rb b/spec/serializers/cluster_entity_spec.rb index 72f02131211..d6a43fd0f00 100644 --- a/spec/serializers/cluster_entity_spec.rb +++ b/spec/serializers/cluster_entity_spec.rb @@ -29,10 +29,23 @@ describe ClusterEntity do context 'when provider type is user' do let(:cluster) { create(:cluster, provider_type: :user) } - it 'has nil' do - expect(subject[:status]).to be_nil + it 'has corresponded data' do + expect(subject[:status]).to eq(:created) expect(subject[:status_reason]).to be_nil end end + + context 'when no application has been installed' do + let(:cluster) { create(:cluster) } + subject { described_class.new(cluster).as_json[:applications]} + + it 'contains helm as not_installable' do + expect(subject).not_to be_empty + + helm = subject[0] + expect(helm[:name]).to eq('helm') + expect(helm[:status]).to eq(:not_installable) + end + end end end diff --git a/spec/serializers/cluster_serializer_spec.rb b/spec/serializers/cluster_serializer_spec.rb index ff7d1789149..5e9f7a45891 100644 --- a/spec/serializers/cluster_serializer_spec.rb +++ b/spec/serializers/cluster_serializer_spec.rb @@ -9,7 +9,7 @@ describe ClusterSerializer do let(:provider) { create(:cluster_provider_gcp, :errored) } it 'serializes only status' do - expect(subject.keys).to contain_exactly(:status, :status_reason) + expect(subject.keys).to contain_exactly(:status, :status_reason, :applications) end end @@ -17,7 +17,7 @@ describe ClusterSerializer do let(:cluster) { create(:cluster, provider_type: :user) } it 'serializes only status' do - expect(subject.keys).to contain_exactly(:status, :status_reason) + expect(subject.keys).to contain_exactly(:status, :status_reason, :applications) end end end diff --git a/spec/services/clusters/applications/check_installation_progress_service_spec.rb b/spec/services/clusters/applications/check_installation_progress_service_spec.rb new file mode 100644 index 00000000000..75fc05d36e9 --- /dev/null +++ b/spec/services/clusters/applications/check_installation_progress_service_spec.rb @@ -0,0 +1,91 @@ +require 'spec_helper' + +describe Clusters::Applications::CheckInstallationProgressService do + RESCHEDULE_PHASES = Gitlab::Kubernetes::Pod::PHASES - [Gitlab::Kubernetes::Pod::SUCCEEDED, Gitlab::Kubernetes::Pod::FAILED].freeze + + let(:application) { create(:cluster_applications_helm, :installing) } + let(:service) { described_class.new(application) } + let(:phase) { Gitlab::Kubernetes::Pod::UNKNOWN } + let(:errors) { nil } + + shared_examples 'a terminated installation' do + it 'removes the installation POD' do + expect(service).to receive(:remove_installation_pod).once + + service.execute + end + end + + shared_examples 'a not yet terminated installation' do |a_phase| + let(:phase) { a_phase } + + context "when phase is #{a_phase}" do + context 'when not timeouted' do + it 'reschedule a new check' do + expect(ClusterWaitForAppInstallationWorker).to receive(:perform_in).once + expect(service).not_to receive(:remove_installation_pod) + + service.execute + + expect(application).to be_installing + expect(application.status_reason).to be_nil + end + end + + context 'when timeouted' do + let(:application) { create(:cluster_applications_helm, :timeouted) } + + it_behaves_like 'a terminated installation' + + it 'make the application errored' do + expect(ClusterWaitForAppInstallationWorker).not_to receive(:perform_in) + + service.execute + + expect(application).to be_errored + expect(application.status_reason).to match(/\btimeouted\b/) + end + end + end + end + + before do + expect(service).to receive(:installation_phase).once.and_return(phase) + + allow(service).to receive(:installation_errors).and_return(errors) + allow(service).to receive(:remove_installation_pod).and_return(nil) + end + + describe '#execute' do + context 'when installation POD succeeded' do + let(:phase) { Gitlab::Kubernetes::Pod::SUCCEEDED } + + it_behaves_like 'a terminated installation' + + it 'make the application installed' do + expect(ClusterWaitForAppInstallationWorker).not_to receive(:perform_in) + + service.execute + + expect(application).to be_installed + expect(application.status_reason).to be_nil + end + end + + context 'when installation POD failed' do + let(:phase) { Gitlab::Kubernetes::Pod::FAILED } + let(:errors) { 'test installation failed' } + + it_behaves_like 'a terminated installation' + + it 'make the application errored' do + service.execute + + expect(application).to be_errored + expect(application.status_reason).to eq(errors) + end + end + + RESCHEDULE_PHASES.each { |phase| it_behaves_like 'a not yet terminated installation', phase } + end +end diff --git a/spec/services/clusters/applications/install_service_spec.rb b/spec/services/clusters/applications/install_service_spec.rb new file mode 100644 index 00000000000..054a49ffedf --- /dev/null +++ b/spec/services/clusters/applications/install_service_spec.rb @@ -0,0 +1,60 @@ +require 'spec_helper' + +describe Clusters::Applications::InstallService do + describe '#execute' do + let(:application) { create(:cluster_applications_helm, :scheduled) } + let(:service) { described_class.new(application) } + let(:helm_client) { instance_double(Gitlab::Kubernetes::Helm) } + + before do + allow(service).to receive(:helm_api).and_return(helm_client) + end + + context 'when there are no errors' do + before do + expect(helm_client).to receive(:install).with(application.install_command) + allow(ClusterWaitForAppInstallationWorker).to receive(:perform_in).and_return(nil) + end + + it 'make the application installing' do + expect(application.cluster).not_to be_nil + service.execute + + expect(application).to be_installing + end + + it 'schedule async installation status check' do + expect(ClusterWaitForAppInstallationWorker).to receive(:perform_in).once + + service.execute + end + end + + context 'when k8s cluster communication fails' do + before do + error = KubeException.new(500, 'system failure', nil) + expect(helm_client).to receive(:install).with(application.install_command).and_raise(error) + end + + it 'make the application errored' do + service.execute + + expect(application).to be_errored + expect(application.status_reason).to match(/kubernetes error:/i) + end + end + + context 'when application cannot be persisted' do + let(:application) { build(:cluster_applications_helm, :scheduled) } + + it 'make the application errored' do + expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid) + expect(helm_client).not_to receive(:install) + + service.execute + + expect(application).to be_errored + end + end + end +end diff --git a/spec/services/clusters/applications/schedule_installation_service_spec.rb b/spec/services/clusters/applications/schedule_installation_service_spec.rb new file mode 100644 index 00000000000..cf95361c935 --- /dev/null +++ b/spec/services/clusters/applications/schedule_installation_service_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +describe Clusters::Applications::ScheduleInstallationService do + def count_scheduled + application_class&.with_status(:scheduled)&.count || 0 + end + + shared_examples 'a failing service' do + it 'raise an exception' do + expect(ClusterInstallAppWorker).not_to receive(:perform_async) + count_before = count_scheduled + + expect { service.execute }.to raise_error(StandardError) + expect(count_scheduled).to eq(count_before) + end + end + + describe '#execute' do + let(:application_class) { Clusters::Applications::Helm } + let(:cluster) { create(:cluster, :project, :provided_by_gcp) } + let(:project) { cluster.project } + let(:service) { described_class.new(project, nil, cluster: cluster, application_class: application_class) } + + it 'creates a new application' do + expect { service.execute }.to change { application_class.count }.by(1) + end + + it 'make the application scheduled' do + expect(ClusterInstallAppWorker).to receive(:perform_async).with(application_class.application_name, kind_of(Numeric)).once + + expect { service.execute }.to change { application_class.with_status(:scheduled).count }.by(1) + end + + context 'when installation is already in progress' do + let(:application) { create(:cluster_applications_helm, :installing) } + let(:cluster) { application.cluster } + + it_behaves_like 'a failing service' + end + + context 'when application_class is nil' do + let(:application_class) { nil } + + it_behaves_like 'a failing service' + end + + context 'when application cannot be persisted' do + before do + expect_any_instance_of(application_class).to receive(:make_scheduled!).once.and_raise(ActiveRecord::RecordInvalid) + end + + it_behaves_like 'a failing service' + end + end +end diff --git a/spec/services/delete_merged_branches_service_spec.rb b/spec/services/delete_merged_branches_service_spec.rb index 5a9eb359ee1..0de02576203 100644 --- a/spec/services/delete_merged_branches_service_spec.rb +++ b/spec/services/delete_merged_branches_service_spec.rb @@ -42,6 +42,14 @@ describe DeleteMergedBranchesService do expect(project.repository.branch_names).to include('improve/awesome') end + it 'ignores protected tags' do + create(:protected_tag, project: project, name: 'improve/*') + + service.execute + + expect(project.repository.branch_names).not_to include('improve/awesome') + end + context 'user without rights' do let(:user) { create(:user) } diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 034065aab00..bf7facaec99 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -6,6 +6,41 @@ describe Projects::ImportService do subject { described_class.new(project, user) } + describe '#async?' do + it 'returns true for an asynchronous importer' do + importer_class = double(:importer, async?: true) + + allow(subject).to receive(:has_importer?).and_return(true) + allow(subject).to receive(:importer_class).and_return(importer_class) + + expect(subject).to be_async + end + + it 'returns false for a regular importer' do + importer_class = double(:importer, async?: false) + + allow(subject).to receive(:has_importer?).and_return(true) + allow(subject).to receive(:importer_class).and_return(importer_class) + + expect(subject).not_to be_async + end + + it 'returns false when the importer does not define #async?' do + importer_class = double(:importer) + + allow(subject).to receive(:has_importer?).and_return(true) + allow(subject).to receive(:importer_class).and_return(importer_class) + + expect(subject).not_to be_async + end + + it 'returns false when the importer does not exist' do + allow(subject).to receive(:has_importer?).and_return(false) + + expect(subject).not_to be_async + end + end + describe '#execute' do context 'with unknown url' do before do @@ -37,21 +72,24 @@ describe Projects::ImportService do end context 'with a Github repository' do - it 'succeeds if repository import is successfully' do - expect_any_instance_of(Github::Import).to receive(:execute).and_return(true) + it 'succeeds if repository import was scheduled' do + expect_any_instance_of(Gitlab::GithubImport::ParallelImporter) + .to receive(:execute) + .and_return(true) result = subject.execute expect(result[:status]).to eq :success end - it 'fails if repository import fails' do - expect_any_instance_of(Repository).to receive(:fetch_remote).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) + it 'fails if repository import was not scheduled' do + expect_any_instance_of(Gitlab::GithubImport::ParallelImporter) + .to receive(:execute) + .and_return(false) result = subject.execute expect(result[:status]).to eq :error - expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.path_with_namespace} - The remote data could not be imported." end end @@ -92,47 +130,22 @@ describe Projects::ImportService do end it 'succeeds if importer succeeds' do - allow_any_instance_of(Github::Import).to receive(:execute).and_return(true) + allow_any_instance_of(Gitlab::GithubImport::ParallelImporter) + .to receive(:execute).and_return(true) result = subject.execute expect(result[:status]).to eq :success end - it 'flushes various caches' do - allow_any_instance_of(Github::Import).to receive(:execute) - .and_return(true) - - expect_any_instance_of(Repository).to receive(:expire_content_cache) - - subject.execute - end - it 'fails if importer fails' do - allow_any_instance_of(Github::Import).to receive(:execute).and_return(false) - - result = subject.execute - - expect(result[:status]).to eq :error - expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - The remote data could not be imported." - end - - it 'fails if importer raise an error' do - allow_any_instance_of(Github::Import).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API')) + allow_any_instance_of(Gitlab::GithubImport::ParallelImporter) + .to receive(:execute) + .and_return(false) result = subject.execute expect(result[:status]).to eq :error - expect(result[:message]).to eq "Error importing repository #{project.import_url} into #{project.full_path} - Github: failed to connect API" - end - - it 'expires content cache after error' do - allow_any_instance_of(Project).to receive(:repository_exists?).and_return(false) - - expect_any_instance_of(Repository).to receive(:fetch_remote).and_raise(Gitlab::Shell::Error.new) - expect_any_instance_of(Repository).to receive(:expire_content_cache) - - subject.execute end end diff --git a/spec/support/controllers/githubish_import_controller_shared_examples.rb b/spec/support/controllers/githubish_import_controller_shared_examples.rb index b23d81a226a..a0839eefe6c 100644 --- a/spec/support/controllers/githubish_import_controller_shared_examples.rb +++ b/spec/support/controllers/githubish_import_controller_shared_examples.rb @@ -14,7 +14,7 @@ shared_examples 'a GitHub-ish import controller: POST personal_access_token' do it "updates access token" do token = 'asdfasdf9876' - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:user).and_return(true) post :personal_access_token, personal_access_token: token @@ -79,7 +79,7 @@ shared_examples 'a GitHub-ish import controller: GET status' do end it "handles an invalid access token" do - allow_any_instance_of(Gitlab::GithubImport::Client) + allow_any_instance_of(Gitlab::LegacyGithubImport::Client) .to receive(:repos).and_raise(Octokit::Unauthorized) get :status @@ -110,7 +110,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do context "when the repository owner is the provider user" do context "when the provider user and GitLab user's usernames match" do it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -122,7 +122,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let(:provider_username) { "someone_else" } it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -149,7 +149,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it "takes the existing namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, existing_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -161,7 +161,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do it "creates a project using user's namespace" do create(:user, username: other_username) - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -173,14 +173,14 @@ shared_examples 'a GitHub-ish import controller: POST create' do context "when a namespace with the provider user's username doesn't exist" do context "when current user can create namespaces" do it "creates the namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).and_return(double(execute: true)) expect { post :create, target_namespace: provider_repo.name, format: :js }.to change(Namespace, :count).by(1) end it "takes the new namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, an_instance_of(Group), user, access_params, type: provider) .and_return(double(execute: true)) @@ -194,14 +194,14 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it "doesn't create the namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).and_return(double(execute: true)) expect { post :create, format: :js }.not_to change(Namespace, :count) end it "takes the current user's namespace" do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, provider_repo.name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -219,7 +219,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, test_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -227,7 +227,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected name and default namespace' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, user.namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -245,7 +245,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, nested_namespace, user, access_params, type: provider) .and_return(double(execute: true)) @@ -257,7 +257,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let(:test_name) { 'test_name' } it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -265,7 +265,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'creates the namespaces' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -274,7 +274,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'new namespace has the right parent' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -289,7 +289,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do let!(:parent_namespace) { create(:group, name: 'foo', owner: user) } it 'takes the selected namespace and name' do - expect(Gitlab::GithubImport::ProjectCreator) + expect(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) @@ -297,7 +297,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do end it 'creates the namespaces' do - allow(Gitlab::GithubImport::ProjectCreator) + allow(Gitlab::LegacyGithubImport::ProjectCreator) .to receive(:new).with(provider_repo, test_name, kind_of(Namespace), user, access_params, type: provider) .and_return(double(execute: true)) diff --git a/spec/support/features/discussion_comments_shared_example.rb b/spec/support/features/discussion_comments_shared_example.rb index aabc64d972b..c24940393f9 100644 --- a/spec/support/features/discussion_comments_shared_example.rb +++ b/spec/support/features/discussion_comments_shared_example.rb @@ -71,7 +71,7 @@ shared_examples 'discussion comments' do |resource_name| expect(page).not_to have_selector menu_selector find(toggle_selector).click - find('body').click + execute_script("document.querySelector('body').click()") expect(page).not_to have_selector menu_selector end diff --git a/spec/support/legacy_path_redirect_shared_examples.rb b/spec/support/legacy_path_redirect_shared_examples.rb new file mode 100644 index 00000000000..f300bdd48b1 --- /dev/null +++ b/spec/support/legacy_path_redirect_shared_examples.rb @@ -0,0 +1,13 @@ +shared_examples 'redirecting a legacy path' do |source, target| + include RSpec::Rails::RequestExampleGroup + + it "redirects #{source} to #{target} when the resource does not exist" do + expect(get(source)).to redirect_to(target) + end + + it "does not redirect #{source} to #{target} when the resource exists" do + resource + + expect(get(source)).not_to redirect_to(target) + end +end diff --git a/spec/support/matchers/access_matchers_for_controller.rb b/spec/support/matchers/access_matchers_for_controller.rb index bb6b7c63ee9..cdb62a5deee 100644 --- a/spec/support/matchers/access_matchers_for_controller.rb +++ b/spec/support/matchers/access_matchers_for_controller.rb @@ -5,7 +5,7 @@ module AccessMatchersForController extend RSpec::Matchers::DSL include Warden::Test::Helpers - EXPECTED_STATUS_CODE_ALLOWED = [200, 201, 302].freeze + EXPECTED_STATUS_CODE_ALLOWED = [200, 201, 204, 302].freeze EXPECTED_STATUS_CODE_DENIED = [401, 404].freeze def emulate_user(role, membership = nil) diff --git a/spec/views/projects/commit/branches.html.haml_spec.rb b/spec/views/projects/commit/branches.html.haml_spec.rb new file mode 100644 index 00000000000..b9d4dc80fe0 --- /dev/null +++ b/spec/views/projects/commit/branches.html.haml_spec.rb @@ -0,0 +1,109 @@ +require 'spec_helper' + +describe 'projects/commit/branches.html.haml' do + let(:project) { create(:project, :repository) } + + before do + assign(:project, project) + end + + context 'when branches and tags are available' do + before do + assign(:branches, ['master', 'test-branch']) + assign(:branches_limit_exceeded, false) + assign(:tags, ['tag1']) + assign(:tags_limit_exceeded, false) + + render + end + + it 'shows default branch' do + expect(rendered).to have_link('master') + end + + it 'shows js expand link' do + expect(rendered).to have_selector('.js-details-expand') + end + + it 'shows branch and tag links' do + expect(rendered).to have_link('test-branch') + expect(rendered).to have_link('tag1') + end + end + + context 'when branches are available but no tags' do + before do + assign(:branches, ['master', 'test-branch']) + assign(:branches_limit_exceeded, false) + assign(:tags, []) + assign(:tags_limit_exceeded, true) + + render + end + + it 'shows branches' do + expect(rendered).to have_link('master') + expect(rendered).to have_link('test-branch') + end + + it 'shows js expand link' do + expect(rendered).to have_selector('.js-details-expand') + end + + it 'shows limit exceeded message for tags' do + expect(rendered).to have_text('Tags unavailable') + end + end + + context 'when tags are available but no branches (just default)' do + before do + assign(:branches, ['master']) + assign(:branches_limit_exceeded, true) + assign(:tags, %w(tag1 tag2)) + assign(:tags_limit_exceeded, false) + + render + end + + it 'shows default branch' do + expect(rendered).to have_text('master') + end + + it 'shows js expand link' do + expect(rendered).to have_selector('.js-details-expand') + end + + it 'shows tags' do + expect(rendered).to have_link('tag1') + expect(rendered).to have_link('tag2') + end + + it 'shows limit exceeded for branches' do + expect(rendered).to have_text('Branches unavailable') + end + end + + context 'when branches and tags are not available' do + before do + assign(:branches, ['master']) + assign(:branches_limit_exceeded, true) + assign(:tags, []) + assign(:tags_limit_exceeded, true) + + render + end + + it 'shows default branch' do + expect(rendered).to have_text('master') + end + + it 'shows js expand link' do + expect(rendered).to have_selector('.js-details-expand') + end + + it 'shows too many to search' do + expect(rendered).to have_text('Branches unavailable') + expect(rendered).to have_text('Tags unavailable') + end + end +end diff --git a/spec/workers/concerns/gitlab/github_import/notify_upon_death_spec.rb b/spec/workers/concerns/gitlab/github_import/notify_upon_death_spec.rb new file mode 100644 index 00000000000..4b9aa9a7ef8 --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/notify_upon_death_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::NotifyUponDeath do + let(:worker_class) do + Class.new do + include Sidekiq::Worker + include Gitlab::GithubImport::NotifyUponDeath + end + end + + describe '.sidekiq_retries_exhausted' do + it 'notifies the JobWaiter when 3 arguments are given and the last is a String' do + job = { 'args' => [12, {}, '123abc'], 'jid' => '123' } + + expect(Gitlab::JobWaiter) + .to receive(:notify) + .with('123abc', '123') + + worker_class.sidekiq_retries_exhausted_block.call(job) + end + + it 'does not notify the JobWaiter when only 2 arguments are given' do + job = { 'args' => [12, {}], 'jid' => '123' } + + expect(Gitlab::JobWaiter) + .not_to receive(:notify) + + worker_class.sidekiq_retries_exhausted_block.call(job) + end + + it 'does not notify the JobWaiter when only 1 argument is given' do + job = { 'args' => [12], 'jid' => '123' } + + expect(Gitlab::JobWaiter) + .not_to receive(:notify) + + worker_class.sidekiq_retries_exhausted_block.call(job) + end + + it 'does not notify the JobWaiter when the last argument is not a String' do + job = { 'args' => [12, {}, 40], 'jid' => '123' } + + expect(Gitlab::JobWaiter) + .not_to receive(:notify) + + worker_class.sidekiq_retries_exhausted_block.call(job) + end + end +end diff --git a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb new file mode 100644 index 00000000000..3ccf06f2d7d --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ObjectImporter do + let(:worker) do + Class.new do + include(Gitlab::GithubImport::ObjectImporter) + + def counter_name + :dummy_counter + end + + def counter_description + 'This is a counter' + end + end.new + end + + describe '#import' do + it 'imports the object' do + representation_class = double(:representation_class) + importer_class = double(:importer_class) + importer_instance = double(:importer_instance) + representation = double(:representation) + project = double(:project, path_with_namespace: 'foo/bar') + client = double(:client) + + expect(worker) + .to receive(:representation_class) + .and_return(representation_class) + + expect(worker) + .to receive(:importer_class) + .and_return(importer_class) + + expect(representation_class) + .to receive(:from_json_hash) + .with(an_instance_of(Hash)) + .and_return(representation) + + expect(importer_class) + .to receive(:new) + .with(representation, project, client) + .and_return(importer_instance) + + expect(importer_instance) + .to receive(:execute) + + expect(worker.counter) + .to receive(:increment) + .with(project: 'foo/bar') + .and_call_original + + worker.import(project, client, { 'number' => 10 }) + end + end + + describe '#counter' do + it 'returns a Prometheus counter' do + expect(worker) + .to receive(:counter_name) + .and_call_original + + expect(worker) + .to receive(:counter_description) + .and_call_original + + worker.counter + end + end +end diff --git a/spec/workers/concerns/gitlab/github_import/queue_spec.rb b/spec/workers/concerns/gitlab/github_import/queue_spec.rb new file mode 100644 index 00000000000..321ae3fe978 --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/queue_spec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Queue do + it 'sets the Sidekiq options for the worker' do + worker = Class.new do + include Sidekiq::Worker + include Gitlab::GithubImport::Queue + end + + expect(worker.sidekiq_options['queue']).to eq('github_importer') + end +end diff --git a/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb new file mode 100644 index 00000000000..8de4059c4ae --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb @@ -0,0 +1,110 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ReschedulingMethods do + let(:worker) do + Class.new { include(Gitlab::GithubImport::ReschedulingMethods) }.new + end + + describe '#perform' do + context 'with a non-existing project' do + it 'does not perform any work' do + expect(worker) + .not_to receive(:try_import) + + worker.perform(-1, {}) + end + + it 'notifies any waiters so they do not wait forever' do + expect(worker) + .to receive(:notify_waiter) + .with('123') + + worker.perform(-1, {}, '123') + end + end + + context 'with an existing project' do + let(:project) { create(:project) } + + it 'notifies any waiters upon successfully importing the data' do + expect(worker) + .to receive(:try_import) + .with( + an_instance_of(Project), + an_instance_of(Gitlab::GithubImport::Client), + { 'number' => 2 } + ) + .and_return(true) + + expect(worker) + .to receive(:notify_waiter).with('123') + + worker.perform(project.id, { 'number' => 2 }, '123') + end + + it 'reschedules itself if the data could not be imported' do + expect(worker) + .to receive(:try_import) + .with( + an_instance_of(Project), + an_instance_of(Gitlab::GithubImport::Client), + { 'number' => 2 } + ) + .and_return(false) + + expect(worker) + .not_to receive(:notify_waiter) + + expect_any_instance_of(Gitlab::GithubImport::Client) + .to receive(:rate_limit_resets_in) + .and_return(14) + + expect(worker.class) + .to receive(:perform_in) + .with(14, project.id, { 'number' => 2 }, '123') + + worker.perform(project.id, { 'number' => 2 }, '123') + end + end + end + + describe '#try_import' do + it 'returns true when the import succeeds' do + expect(worker) + .to receive(:import) + .with(10, 20) + + expect(worker.try_import(10, 20)).to eq(true) + end + + it 'returns false when the import fails due to hitting the GitHub API rate limit' do + expect(worker) + .to receive(:import) + .with(10, 20) + .and_raise(Gitlab::GithubImport::RateLimitError) + + expect(worker.try_import(10, 20)).to eq(false) + end + end + + describe '#notify_waiter' do + it 'notifies the waiter if a waiter key is specified' do + expect(worker) + .to receive(:jid) + .and_return('abc123') + + expect(Gitlab::JobWaiter) + .to receive(:notify) + .with('123', 'abc123') + + worker.notify_waiter('123') + end + + it 'does not notify any waiters if no waiter key is specified' do + expect(Gitlab::JobWaiter) + .not_to receive(:notify) + + worker.notify_waiter(nil) + end + end +end diff --git a/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb new file mode 100644 index 00000000000..241e8a2b6d3 --- /dev/null +++ b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::StageMethods do + let(:project) { create(:project) } + let(:worker) do + Class.new { include(Gitlab::GithubImport::StageMethods) }.new + end + + describe '#perform' do + it 'returns if no project could be found' do + expect(worker).not_to receive(:try_import) + + worker.perform(-1) + end + + it 'imports the data when the project exists' do + allow(worker) + .to receive(:find_project) + .with(project.id) + .and_return(project) + + expect(worker) + .to receive(:try_import) + .with( + an_instance_of(Gitlab::GithubImport::Client), + an_instance_of(Project) + ) + + worker.perform(project.id) + end + end + + describe '#try_import' do + it 'imports the project' do + client = double(:client) + + expect(worker) + .to receive(:import) + .with(client, project) + + worker.try_import(client, project) + end + + it 'reschedules the worker if RateLimitError was raised' do + client = double(:client, rate_limit_resets_in: 10) + + expect(worker) + .to receive(:import) + .with(client, project) + .and_raise(Gitlab::GithubImport::RateLimitError) + + expect(worker.class) + .to receive(:perform_in) + .with(10, project.id) + + worker.try_import(client, project) + end + end + + describe '#find_project' do + it 'returns a Project for an existing ID' do + project.update_column(:import_status, 'started') + + expect(worker.find_project(project.id)).to eq(project) + end + + it 'returns nil for a project that failed importing' do + project.update_column(:import_status, 'failed') + + expect(worker.find_project(project.id)).to be_nil + end + + it 'returns nil for a non-existing project ID' do + expect(worker.find_project(-1)).to be_nil + end + end +end diff --git a/spec/workers/gitlab/github_import/advance_stage_worker_spec.rb b/spec/workers/gitlab/github_import/advance_stage_worker_spec.rb new file mode 100644 index 00000000000..3be49a0dee8 --- /dev/null +++ b/spec/workers/gitlab/github_import/advance_stage_worker_spec.rb @@ -0,0 +1,115 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::AdvanceStageWorker, :clean_gitlab_redis_shared_state do + let(:project) { create(:project, import_jid: '123') } + let(:worker) { described_class.new } + + describe '#perform' do + context 'when the project no longer exists' do + it 'does not perform any work' do + expect(worker).not_to receive(:wait_for_jobs) + + worker.perform(-1, { '123' => 2 }, :finish) + end + end + + context 'when there are remaining jobs' do + before do + allow(worker) + .to receive(:find_project) + .and_return(project) + end + + it 'reschedules itself' do + expect(worker) + .to receive(:wait_for_jobs) + .with({ '123' => 2 }) + .and_return({ '123' => 1 }) + + expect(described_class) + .to receive(:perform_in) + .with(described_class::INTERVAL, project.id, { '123' => 1 }, :finish) + + worker.perform(project.id, { '123' => 2 }, :finish) + end + end + + context 'when there are no remaining jobs' do + before do + allow(worker) + .to receive(:find_project) + .and_return(project) + + allow(worker) + .to receive(:wait_for_jobs) + .with({ '123' => 2 }) + .and_return({}) + end + + it 'schedules the next stage' do + expect(project) + .to receive(:refresh_import_jid_expiration) + + expect(Gitlab::GithubImport::Stage::FinishImportWorker) + .to receive(:perform_async) + .with(project.id) + + worker.perform(project.id, { '123' => 2 }, :finish) + end + + it 'raises KeyError when the stage name is invalid' do + expect { worker.perform(project.id, { '123' => 2 }, :kittens) } + .to raise_error(KeyError) + end + end + end + + describe '#wait_for_jobs' do + it 'waits for jobs to complete and returns a new pair of keys to wait for' do + waiter1 = double(:waiter1, jobs_remaining: 1, key: '123') + waiter2 = double(:waiter2, jobs_remaining: 0, key: '456') + + expect(Gitlab::JobWaiter) + .to receive(:new) + .ordered + .with(2, '123') + .and_return(waiter1) + + expect(Gitlab::JobWaiter) + .to receive(:new) + .ordered + .with(1, '456') + .and_return(waiter2) + + expect(waiter1) + .to receive(:wait) + .with(described_class::BLOCKING_WAIT_TIME) + + expect(waiter2) + .to receive(:wait) + .with(described_class::BLOCKING_WAIT_TIME) + + new_waiters = worker.wait_for_jobs({ '123' => 2, '456' => 1 }) + + expect(new_waiters).to eq({ '123' => 1 }) + end + end + + describe '#find_project' do + it 'returns a Project' do + project.update_column(:import_status, 'started') + + found = worker.find_project(project.id) + + expect(found).to be_an_instance_of(Project) + + # This test is there to make sure we only select the columns we care + # about. + expect(found.attributes).to eq({ 'id' => nil, 'import_jid' => '123' }) + end + + it 'returns nil if the project import is not running' do + expect(worker.find_project(project.id)).to be_nil + end + end +end diff --git a/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb new file mode 100644 index 00000000000..7c8c665a9b3 --- /dev/null +++ b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ImportDiffNoteWorker do + let(:worker) { described_class.new } + + describe '#import' do + it 'imports a diff note' do + project = double(:project, path_with_namespace: 'foo/bar') + client = double(:client) + importer = double(:importer) + hash = { + 'noteable_id' => 42, + 'path' => 'README.md', + 'commit_id' => '123abc', + 'diff_hunk' => "@@ -1 +1 @@\n-Hello\n+Hello world", + 'user' => { 'id' => 4, 'login' => 'alice' }, + 'note' => 'Hello world', + 'created_at' => Time.zone.now.to_s, + 'updated_at' => Time.zone.now.to_s + } + + expect(Gitlab::GithubImport::Importer::DiffNoteImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::DiffNote), + project, + client + ) + .and_return(importer) + + expect(importer) + .to receive(:execute) + + expect(worker.counter) + .to receive(:increment) + .with(project: 'foo/bar') + .and_call_original + + worker.import(project, client, hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/import_issue_worker_spec.rb b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb new file mode 100644 index 00000000000..4116380ff4d --- /dev/null +++ b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ImportIssueWorker do + let(:worker) { described_class.new } + + describe '#import' do + it 'imports an issue' do + project = double(:project, path_with_namespace: 'foo/bar') + client = double(:client) + importer = double(:importer) + hash = { + 'iid' => 42, + 'title' => 'My Issue', + 'description' => 'This is my issue', + 'milestone_number' => 4, + 'state' => 'opened', + 'assignees' => [{ 'id' => 4, 'login' => 'alice' }], + 'label_names' => %w[bug], + 'user' => { 'id' => 4, 'login' => 'alice' }, + 'created_at' => Time.zone.now.to_s, + 'updated_at' => Time.zone.now.to_s, + 'pull_request' => false + } + + expect(Gitlab::GithubImport::Importer::IssueAndLabelLinksImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Issue), + project, + client + ) + .and_return(importer) + + expect(importer) + .to receive(:execute) + + expect(worker.counter) + .to receive(:increment) + .with(project: 'foo/bar') + .and_call_original + + worker.import(project, client, hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/import_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_note_worker_spec.rb new file mode 100644 index 00000000000..0ca825a722b --- /dev/null +++ b/spec/workers/gitlab/github_import/import_note_worker_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ImportNoteWorker do + let(:worker) { described_class.new } + + describe '#import' do + it 'imports a note' do + project = double(:project, path_with_namespace: 'foo/bar') + client = double(:client) + importer = double(:importer) + hash = { + 'noteable_id' => 42, + 'noteable_type' => 'issues', + 'user' => { 'id' => 4, 'login' => 'alice' }, + 'note' => 'Hello world', + 'created_at' => Time.zone.now.to_s, + 'updated_at' => Time.zone.now.to_s + } + + expect(Gitlab::GithubImport::Importer::NoteImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::Note), + project, + client + ) + .and_return(importer) + + expect(importer) + .to receive(:execute) + + expect(worker.counter) + .to receive(:increment) + .with(project: 'foo/bar') + .and_call_original + + worker.import(project, client, hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb new file mode 100644 index 00000000000..d49f560af42 --- /dev/null +++ b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb @@ -0,0 +1,51 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::ImportPullRequestWorker do + let(:worker) { described_class.new } + + describe '#import' do + it 'imports a pull request' do + project = double(:project, path_with_namespace: 'foo/bar') + client = double(:client) + importer = double(:importer) + hash = { + 'iid' => 42, + 'title' => 'My Pull Request', + 'description' => 'This is my pull request', + 'source_branch' => 'my-feature', + 'source_branch_sha' => '123abc', + 'target_branch' => 'master', + 'target_branch_sha' => '456def', + 'source_repository_id' => 400, + 'target_repository_id' => 200, + 'source_repository_owner' => 'alice', + 'state' => 'closed', + 'milestone_number' => 4, + 'user' => { 'id' => 4, 'login' => 'alice' }, + 'assignee' => { 'id' => 4, 'login' => 'alice' }, + 'created_at' => Time.zone.now.to_s, + 'updated_at' => Time.zone.now.to_s, + 'merged_at' => Time.zone.now.to_s + } + + expect(Gitlab::GithubImport::Importer::PullRequestImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::PullRequest), + project, + client + ) + .and_return(importer) + + expect(importer) + .to receive(:execute) + + expect(worker.counter) + .to receive(:increment) + .with(project: 'foo/bar') + .and_call_original + + worker.import(project, client, hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/refresh_import_jid_worker_spec.rb b/spec/workers/gitlab/github_import/refresh_import_jid_worker_spec.rb new file mode 100644 index 00000000000..073c6d7a2f5 --- /dev/null +++ b/spec/workers/gitlab/github_import/refresh_import_jid_worker_spec.rb @@ -0,0 +1,95 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::RefreshImportJidWorker do + let(:worker) { described_class.new } + + describe '.perform_in_the_future' do + it 'schedules a job in the future' do + expect(described_class) + .to receive(:perform_in) + .with(1.minute.to_i, 10, '123') + + described_class.perform_in_the_future(10, '123') + end + end + + describe '#perform' do + let(:project) { create(:project, import_jid: '123abc') } + + context 'when the project does not exist' do + it 'does nothing' do + expect(Gitlab::SidekiqStatus) + .not_to receive(:running?) + + worker.perform(-1, '123') + end + end + + context 'when the job is running' do + it 'refreshes the import JID and reschedules itself' do + allow(worker) + .to receive(:find_project) + .with(project.id) + .and_return(project) + + expect(Gitlab::SidekiqStatus) + .to receive(:running?) + .with('123') + .and_return(true) + + expect(project) + .to receive(:refresh_import_jid_expiration) + + expect(worker.class) + .to receive(:perform_in_the_future) + .with(project.id, '123') + + worker.perform(project.id, '123') + end + end + + context 'when the job is no longer running' do + it 'returns' do + allow(worker) + .to receive(:find_project) + .with(project.id) + .and_return(project) + + expect(Gitlab::SidekiqStatus) + .to receive(:running?) + .with('123') + .and_return(false) + + expect(project) + .not_to receive(:refresh_import_jid_expiration) + + worker.perform(project.id, '123') + end + end + end + + describe '#find_project' do + it 'returns a Project' do + project = create(:project, import_status: 'started') + + expect(worker.find_project(project.id)).to be_an_instance_of(Project) + end + + it 'only selects the import JID field' do + project = create(:project, import_status: 'started', import_jid: '123abc') + + expect(worker.find_project(project.id).attributes) + .to eq({ 'id' => nil, 'import_jid' => '123abc' }) + end + + it 'returns nil for a project for which the import process failed' do + project = create(:project, import_status: 'failed') + + expect(worker.find_project(project.id)).to be_nil + end + + it 'returns nil for a non-existing project' do + expect(worker.find_project(-1)).to be_nil + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb new file mode 100644 index 00000000000..91e0cddb5d8 --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::FinishImportWorker do + let(:project) { create(:project) } + let(:worker) { described_class.new } + + describe '#perform' do + it 'marks the import as finished' do + expect(project).to receive(:after_import) + expect(worker).to receive(:report_import_time).with(project) + + worker.import(double(:client), project) + end + end + + describe '#report_import_time' do + it 'reports the total import time' do + expect(worker.histogram) + .to receive(:observe) + .with({ project: project.path_with_namespace }, a_kind_of(Numeric)) + .and_call_original + + expect(worker.counter) + .to receive(:increment) + .and_call_original + + expect(worker.logger).to receive(:info).with(an_instance_of(String)) + + worker.report_import_time(project) + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/import_base_data_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_base_data_worker_spec.rb new file mode 100644 index 00000000000..8c80d660287 --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/import_base_data_worker_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::ImportBaseDataWorker do + let(:project) { create(:project) } + let(:worker) { described_class.new } + + describe '#import' do + it 'imports the base data of a project' do + importer = double(:importer) + client = double(:client) + + described_class::IMPORTERS.each do |klass| + expect(klass) + .to receive(:new) + .with(project, client) + .and_return(importer) + + expect(importer).to receive(:execute) + end + + expect(project).to receive(:refresh_import_jid_expiration) + + expect(Gitlab::GithubImport::Stage::ImportPullRequestsWorker) + .to receive(:perform_async) + .with(project.id) + + worker.import(client, project) + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker_spec.rb new file mode 100644 index 00000000000..ab347f5b75b --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::ImportIssuesAndDiffNotesWorker do + let(:project) { create(:project) } + let(:worker) { described_class.new } + + describe '#import' do + it 'imports the issues and diff notes' do + client = double(:client) + + described_class::IMPORTERS.each do |klass| + importer = double(:importer) + waiter = Gitlab::JobWaiter.new(2, '123') + + expect(klass) + .to receive(:new) + .with(project, client) + .and_return(importer) + + expect(importer) + .to receive(:execute) + .and_return(waiter) + end + + expect(Gitlab::GithubImport::AdvanceStageWorker) + .to receive(:perform_async) + .with(project.id, { '123' => 2 }, :notes) + + worker.import(client, project) + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/import_notes_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_notes_worker_spec.rb new file mode 100644 index 00000000000..098d2d55386 --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/import_notes_worker_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::ImportNotesWorker do + let(:project) { create(:project) } + let(:worker) { described_class.new } + + describe '#import' do + it 'imports all the notes' do + importer = double(:importer) + client = double(:client) + waiter = Gitlab::JobWaiter.new(2, '123') + + expect(Gitlab::GithubImport::Importer::NotesImporter) + .to receive(:new) + .with(project, client) + .and_return(importer) + + expect(importer) + .to receive(:execute) + .and_return(waiter) + + expect(Gitlab::GithubImport::AdvanceStageWorker) + .to receive(:perform_async) + .with(project.id, { '123' => 2 }, :finish) + + worker.import(client, project) + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/import_pull_requests_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_pull_requests_worker_spec.rb new file mode 100644 index 00000000000..2fc91a3e80a --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/import_pull_requests_worker_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::ImportPullRequestsWorker do + let(:project) { create(:project) } + let(:worker) { described_class.new } + + describe '#import' do + it 'imports all the pull requests' do + importer = double(:importer) + client = double(:client) + waiter = Gitlab::JobWaiter.new(2, '123') + + expect(Gitlab::GithubImport::Importer::PullRequestsImporter) + .to receive(:new) + .with(project, client) + .and_return(importer) + + expect(importer) + .to receive(:execute) + .and_return(waiter) + + expect(project) + .to receive(:refresh_import_jid_expiration) + + expect(Gitlab::GithubImport::AdvanceStageWorker) + .to receive(:perform_async) + .with(project.id, { '123' => 2 }, :issues_and_diff_notes) + + worker.import(client, project) + end + end +end diff --git a/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb new file mode 100644 index 00000000000..adab535ac05 --- /dev/null +++ b/spec/workers/gitlab/github_import/stage/import_repository_worker_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +describe Gitlab::GithubImport::Stage::ImportRepositoryWorker do + let(:project) { double(:project, id: 4) } + let(:worker) { described_class.new } + + describe '#import' do + before do + expect(Gitlab::GithubImport::RefreshImportJidWorker) + .to receive(:perform_in_the_future) + .with(project.id, '123') + + expect(worker) + .to receive(:jid) + .and_return('123') + end + + context 'when the import succeeds' do + it 'schedules the importing of the base data' do + client = double(:client) + + expect_any_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) + .to receive(:execute) + .and_return(true) + + expect(Gitlab::GithubImport::Stage::ImportBaseDataWorker) + .to receive(:perform_async) + .with(project.id) + + worker.import(client, project) + end + end + + context 'when the import fails' do + it 'does not schedule the importing of the base data' do + client = double(:client) + + expect_any_instance_of(Gitlab::GithubImport::Importer::RepositoryImporter) + .to receive(:execute) + .and_return(false) + + expect(Gitlab::GithubImport::Stage::ImportBaseDataWorker) + .not_to receive(:perform_async) + + worker.import(client, project) + end + end + end +end diff --git a/spec/workers/repository_import_worker_spec.rb b/spec/workers/repository_import_worker_spec.rb index 5cff5108477..0af537647ad 100644 --- a/spec/workers/repository_import_worker_spec.rb +++ b/spec/workers/repository_import_worker_spec.rb @@ -59,5 +59,28 @@ describe RepositoryImportWorker do expect(project.reload.import_status).to eq('failed') end end + + context 'when using an asynchronous importer' do + it 'does not mark the import process as finished' do + service = double(:service) + + allow(Projects::ImportService) + .to receive(:new) + .and_return(service) + + allow(service) + .to receive(:execute) + .and_return(true) + + allow(service) + .to receive(:async?) + .and_return(true) + + expect_any_instance_of(Project) + .not_to receive(:import_finish) + + subject.perform(project.id) + end + end end end diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000000..3a569cf7c19 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "target": "es5", + "strict": true, + "module": "es2015", + "moduleResolution": "node" + } +}
\ No newline at end of file diff --git a/vendor/gitignore/Composer.gitignore b/vendor/gitignore/Composer.gitignore index c4222678424..a67d42b32f8 100644 --- a/vendor/gitignore/Composer.gitignore +++ b/vendor/gitignore/Composer.gitignore @@ -1,6 +1,6 @@ composer.phar /vendor/ -# Commit your application's lock file http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file +# Commit your application's lock file https://getcomposer.org/doc/01-basic-usage.md#commit-your-composer-lock-file-to-version-control # You may choose to ignore a library lock file http://getcomposer.org/doc/02-libraries.md#lock-file # composer.lock diff --git a/vendor/gitignore/Global/Windows.gitignore b/vendor/gitignore/Global/Windows.gitignore index dff26a9ab70..846a1db836c 100644 --- a/vendor/gitignore/Global/Windows.gitignore +++ b/vendor/gitignore/Global/Windows.gitignore @@ -7,7 +7,7 @@ ehthumbs_vista.db *.stackdump # Folder config file -Desktop.ini +[Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ diff --git a/vendor/gitignore/Terraform.gitignore b/vendor/gitignore/Terraform.gitignore index 9b5aebb1b35..1fef4ab91e1 100644 --- a/vendor/gitignore/Terraform.gitignore +++ b/vendor/gitignore/Terraform.gitignore @@ -1,10 +1,9 @@ -# Compiled files -*.tfstate -*.tfstate.*.backup -*.tfstate.backup +# Local .terraform directories +**/.terraform/* -# Module directory -.terraform/ +# .tfstate files +*.tfstate +*.tfstate.* -# Variable values for development -terraform.tfvars +# .tfvars files +*.tfvars diff --git a/vendor/gitignore/VisualStudio.gitignore b/vendor/gitignore/VisualStudio.gitignore index 0867ec5a7ee..509668db67a 100644 --- a/vendor/gitignore/VisualStudio.gitignore +++ b/vendor/gitignore/VisualStudio.gitignore @@ -171,11 +171,11 @@ PublishScripts/ # NuGet Packages *.nupkg # The packages folder can be ignored because of Package Restore -**/packages/* +**/[Pp]ackages/* # except build/, which is used as an MSBuild target. -!**/packages/build/ +!**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed -#!**/packages/repositories.config +#!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml index c93e6567baf..88261502d7f 100644 --- a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml @@ -335,7 +335,9 @@ production: function check_kube_domain() { if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then - echo "In order to deploy, AUTO_DEVOPS_DOMAIN must be set as a variable at the group or project level, or manually added in .gitlab-cy.yml" + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a secret variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" false else true diff --git a/vendor/gitlab-ci-yml/OpenShift.gitlab-ci.yml b/vendor/gitlab-ci-yml/OpenShift.gitlab-ci.yml index 636cb0a9a99..290b9997084 100644 --- a/vendor/gitlab-ci-yml/OpenShift.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/OpenShift.gitlab-ci.yml @@ -31,14 +31,14 @@ test2: - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID" script: - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker" - - "oc start-build $APP --from-dir=. --follow || sleep 3s || oc start-build $APP --from-dir=. --follow" + - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow" - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST" review: <<: *deploy stage: review variables: - APP: $CI_COMMIT_REF_NAME + APP: review-$CI_COMMIT_REF_NAME APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN environment: name: review/$CI_COMMIT_REF_NAME @@ -56,7 +56,7 @@ stop-review: - oc delete all -l "app=$APP" when: manual variables: - APP: $CI_COMMIT_REF_NAME + APP: review-$CI_COMMIT_REF_NAME GIT_STRATEGY: none environment: name: review/$CI_COMMIT_REF_NAME diff --git a/vendor/gitlab-ci-yml/Rust.gitlab-ci.yml b/vendor/gitlab-ci-yml/Rust.gitlab-ci.yml index 7810121c350..6573eceaa59 100644 --- a/vendor/gitlab-ci-yml/Rust.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/Rust.gitlab-ci.yml @@ -1,6 +1,6 @@ -# Unofficial language image. Look for the different tagged releases at: -# https://hub.docker.com/r/scorpil/rust/tags/ -image: "scorpil/rust:stable" +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/rust/tags/ +image: "rust:latest" # Optional: Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. diff --git a/vendor/gitlab-ci-yml/dotNET.gitlab-ci.yml b/vendor/gitlab-ci-yml/dotNET.gitlab-ci.yml new file mode 100644 index 00000000000..fc3d4ecdbba --- /dev/null +++ b/vendor/gitlab-ci-yml/dotNET.gitlab-ci.yml @@ -0,0 +1,86 @@ +# The following script will work for any project that can be built from command line by msbuild +# It uses powershell shell executor, so you need to add the following line to your config.toml file +# (located in gitlab-runner.exe directory): +# shell = "powershell" +# +# The script is composed of 3 stages: build, test and deploy. +# +# The build stage restores NuGet packages and uses msbuild to build the exe and msi +# One major issue you'll find is that you can't build msi projects from command line +# if you use vdproj. There are workarounds building msi via devenv, but they rarely work +# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# by msbuild. +# +# The test stage runs nunit from command line against Test project inside your solution +# It also saves the resulting TestResult.xml file +# +# The deploy stage copies the exe and msi from build stage to a network drive +# You need to have the network drive mapped as Local System user for gitlab-runner service to see it +# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes + + +# place project specific paths in variables to make the rest of the script more generic +variables: + EXE_RELEASE_FOLDER: 'YourApp\bin\Release' + MSI_RELEASE_FOLDER: 'Setup\bin\Release' + TEST_FOLDER: 'Tests\bin\Release' + DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' + + NUGET_PATH: 'C:\NuGet\nuget.exe' + MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' + NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' + +stages: + - build + - test + - deploy + +build_job: + stage: build + only: + - tags # the build process will only be started by git tag commits + script: + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + +test_job: + stage: test + only: + - tags + script: + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + dependencies: + - build_job + +deploy_job: + stage: deploy + only: + - tags + script: + # Compose a folder for each release based on commit tag. + # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit' + # the artifact files will be copied to: + # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ + - '$commitSubject = git log -1 --pretty=%s' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + + # xcopy takes care of recursively creating required folders + - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' + - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' + - 'xcopy /y ".\TestResult.xml" "$deployFolder"' + + dependencies: + - build_job + - test_job +
\ No newline at end of file diff --git a/yarn.lock b/yarn.lock index bf92370d44f..efae3892c31 100644 --- a/yarn.lock +++ b/yarn.lock @@ -101,6 +101,12 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" +ansi-styles@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88" + dependencies: + color-convert "^1.9.0" + anymatch@^1.3.0: version "1.3.2" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-1.3.2.tgz#553dcb8f91e3c889845dfdba34c77721b90b9d7a" @@ -217,18 +223,12 @@ async@1.x, async@^1.4.0, async@^1.4.2, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" -async@2.4.1: +async@2.4.1, async@^2.1.2, async@^2.1.4: version "2.4.1" resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7" dependencies: lodash "^4.14.0" -async@^2.1.2, async@^2.1.4: - version "2.5.0" - resolved "https://registry.yarnpkg.com/async/-/async-2.5.0.tgz#843190fd6b7357a0b9e1c956edddd5ec8462b54d" - dependencies: - lodash "^4.14.0" - async@~0.9.0: version "0.9.2" resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d" @@ -267,7 +267,7 @@ axios@^0.16.2: follow-redirects "^1.2.3" is-buffer "^1.1.5" -babel-code-frame@^6.11.0, babel-code-frame@^6.22.0: +babel-code-frame@^6.11.0, babel-code-frame@^6.16.0, babel-code-frame@^6.22.0: version "6.22.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.22.0.tgz#027620bee567a88c32561574e7fd0801d33118e4" dependencies: @@ -275,14 +275,6 @@ babel-code-frame@^6.11.0, babel-code-frame@^6.22.0: esutils "^2.0.2" js-tokens "^3.0.0" -babel-code-frame@^6.16.0: - version "6.26.0" - resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" - dependencies: - chalk "^1.1.3" - esutils "^2.0.2" - js-tokens "^3.0.2" - babel-core@^6.22.1, babel-core@^6.23.0: version "6.23.1" resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.23.1.tgz#c143cb621bb2f621710c220c5d579d15b8a442df" @@ -932,11 +924,7 @@ bluebird@^2.10.2: version "2.11.0" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1" -bluebird@^3.0.5, bluebird@^3.1.1: - version "3.4.7" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" - -bluebird@^3.3.0: +bluebird@^3.0.5, bluebird@^3.1.1, bluebird@^3.3.0: version "3.5.0" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" @@ -1067,10 +1055,6 @@ buffer-indexof@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.0.tgz#f54f647c4f4e25228baa656a2e57e43d5f270982" -buffer-shims@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-xor@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -1170,6 +1154,14 @@ chalk@1.1.3, chalk@^1.0.0, chalk@^1.1.0, chalk@^1.1.1, chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +chalk@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba" + dependencies: + ansi-styles "^3.1.0" + escape-string-regexp "^1.0.5" + supports-color "^4.0.0" + chokidar@^1.4.1, chokidar@^1.4.3, chokidar@^1.6.0, chokidar@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468" @@ -1253,7 +1245,7 @@ code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" -color-convert@^1.3.0: +color-convert@^1.3.0, color-convert@^1.9.0: version "1.9.0" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.0.tgz#1accf97dd739b983bf994d56fec8f95853641b7a" dependencies: @@ -1454,11 +1446,7 @@ copy-webpack-plugin@^4.0.1: minimatch "^3.0.0" node-dir "^0.1.10" -core-js@^2.2.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.0.tgz#569c050918be6486b3837552028ae0466b717086" - -core-js@^2.4.0, core-js@^2.4.1: +core-js@^2.2.0, core-js@^2.4.0, core-js@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e" @@ -1997,7 +1985,7 @@ engine.io@1.8.3: engine.io-parser "1.3.2" ws "1.1.2" -enhanced-resolve@^3.4.0: +enhanced-resolve@^3.0.0, enhanced-resolve@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-3.4.1.tgz#0421e339fd71419b3da13d129b3979040230476e" dependencies: @@ -2078,11 +2066,7 @@ es6-map@^0.1.3: es6-symbol "~3.1.1" event-emitter "~0.3.5" -es6-promise@^3.0.2: - version "3.3.1" - resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613" - -es6-promise@~3.0.2: +es6-promise@^3.0.2, es6-promise@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.0.2.tgz#010d5858423a5f118979665f46486a95c6ee2bb6" @@ -2263,10 +2247,6 @@ esprima@2.7.x, esprima@^2.6.0, esprima@^2.7.1: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" -esprima@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" - esprima@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" @@ -2667,11 +2647,11 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" -function-bind@^1.0.2: +function-bind@^1.0.2, function-bind@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771" -function-bind@^1.1.1, function-bind@~1.1.0: +function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -2757,25 +2737,25 @@ glob@^6.0.4: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.0, glob@^7.1.1, glob@~7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" +glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.4" + minimatch "^3.0.2" once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.3, glob@^7.0.5: - version "7.1.1" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" +glob@~7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "^3.0.2" + minimatch "^3.0.4" once "^1.3.0" path-is-absolute "^1.0.0" @@ -2978,14 +2958,10 @@ html-comment-regex@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.1.tgz#668b93776eaae55ebde8f3ad464b307a4963625e" -html-entities@1.2.0: +html-entities@1.2.0, html-entities@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.2.0.tgz#41948caf85ce82fed36e4e6a0ed371a6664379e2" -html-entities@^1.2.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.2.1.tgz#0df29351f0721163515dfb9e5543e5f6eed5162f" - htmlparser2@^3.8.2: version "3.9.2" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338" @@ -3391,10 +3367,6 @@ isexe@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/isexe/-/isexe-1.1.2.tgz#36f3e22e60750920f5e7241a476a8c6a42275ad0" -isexe@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" - isobject@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" @@ -3538,18 +3510,7 @@ js-tokens@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.1.tgz#08e9f132484a2c45a30907e9dc4d5567b7f114d7" -js-tokens@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" - -js-yaml@3.x, js-yaml@^3.4.3, js-yaml@^3.7.0: - version "3.8.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.1.tgz#782ba50200be7b9e5a8537001b7804db3ad02628" - dependencies: - argparse "^1.0.7" - esprima "^3.1.1" - -js-yaml@^3.5.1: +js-yaml@3.x, js-yaml@^3.4.3, js-yaml@^3.5.1, js-yaml@^3.7.0: version "3.9.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" dependencies: @@ -4090,7 +4051,7 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" -"mime-db@>= 1.29.0 < 2", mime-db@~1.29.0: +"mime-db@>= 1.29.0 < 2": version "1.29.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" @@ -4098,13 +4059,7 @@ mime-db@~1.27.0: version "1.27.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1" -mime-types@^2.1.12, mime-types@~2.1.7: - version "2.1.16" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" - dependencies: - mime-db "~1.29.0" - -mime-types@~2.1.11, mime-types@~2.1.15: +mime-types@^2.1.12, mime-types@~2.1.11, mime-types@~2.1.15, mime-types@~2.1.7: version "2.1.15" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed" dependencies: @@ -4328,16 +4283,7 @@ nopt@~1.0.10: dependencies: abbrev "1" -normalize-package-data@^2.3.2: - version "2.3.5" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.5.tgz#8d924f142960e1777e7ffe170543631cc7cb02df" - dependencies: - hosted-git-info "^2.1.4" - is-builtin-module "^1.0.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -normalize-package-data@^2.3.4: +normalize-package-data@^2.3.2, normalize-package-data@^2.3.4: version "2.4.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: @@ -4512,7 +4458,7 @@ os-locale@^2.0.0: lcid "^1.0.0" mem "^1.1.0" -os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2: +os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" @@ -5208,7 +5154,7 @@ read-pkg@^2.0.0: normalize-package-data "^2.3.2" path-type "^2.0.0" -readable-stream@^2.0.0, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.2.2, readable-stream@^2.2.9: +readable-stream@^2.0.0, readable-stream@^2.0.6, readable-stream@^2.1.0, readable-stream@^2.1.4, readable-stream@^2.2.2, readable-stream@^2.2.9: version "2.3.3" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: @@ -5231,18 +5177,6 @@ readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable string_decoder "~0.10.x" util-deprecate "~1.0.1" -readable-stream@^2.1.0: - version "2.2.2" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.2.tgz#a9e6fec3c7dda85f8bb1b3ba7028604556fc825e" - dependencies: - buffer-shims "^1.0.0" - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "~1.0.0" - process-nextick-args "~1.0.6" - string_decoder "~0.10.x" - util-deprecate "~1.0.1" - readable-stream@~1.0.2: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -5528,14 +5462,10 @@ semver-diff@^2.0.0: dependencies: semver "^5.0.3" -"semver@2 || 3 || 4 || 5", semver@^5.3.0: +"semver@2 || 3 || 4 || 5", semver@^5.0.1, semver@^5.0.3, semver@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" -semver@^5.0.3: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" - semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -5943,7 +5873,7 @@ supports-color@^3.1.0, supports-color@^3.1.1, supports-color@^3.1.2, supports-co dependencies: has-flag "^1.0.0" -supports-color@^4.2.1: +supports-color@^4.0.0, supports-color@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.2.1.tgz#65a4bb2631e90e02420dba5554c375a4754bb836" dependencies: @@ -6057,10 +5987,6 @@ thunky@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/thunky/-/thunky-0.1.0.tgz#bf30146824e2b6e67b0f2d7a4ac8beb26908684e" -time-stamp@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/time-stamp/-/time-stamp-2.0.0.tgz#95c6a44530e15ba8d6f4a3ecb8c3a3fac46da357" - timeago.js@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/timeago.js/-/timeago.js-2.0.5.tgz#730c74fbdb0b0917a553675a4460e3a7f80db86c" @@ -6089,18 +6015,12 @@ tiny-emitter@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-1.1.0.tgz#ab405a21ffed814a76c19739648093d70654fecb" -tmp@0.0.31: +tmp@0.0.31, tmp@0.0.x: version "0.0.31" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.31.tgz#8f38ab9438e17315e5dbd8b3657e8bfb277ae4a7" dependencies: os-tmpdir "~1.0.1" -tmp@0.0.x: - version "0.0.33" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" - dependencies: - os-tmpdir "~1.0.2" - to-array@0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/to-array/-/to-array-0.1.4.tgz#17e6c11f73dd4f3d74cda7a4ff3238e9ad9bf890" @@ -6141,6 +6061,15 @@ tryit@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/tryit/-/tryit-1.0.3.tgz#393be730a9446fd1ead6da59a014308f36c289cb" +ts-loader@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-3.1.1.tgz#602d93c12029eaf8fa1ee478a90785d40c5f6658" + dependencies: + chalk "^2.3.0" + enhanced-resolve "^3.0.0" + loader-utils "^1.0.2" + semver "^5.0.1" + tty-browserify@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6" @@ -6172,6 +6101,10 @@ typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" +typescript@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.6.1.tgz#ef39cdea27abac0b500242d6726ab90e0c846631" + uglify-js@^2.6, uglify-js@^2.8.29: version "2.8.29" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" @@ -6447,7 +6380,7 @@ webpack-bundle-analyzer@^2.8.2: opener "^1.4.3" ws "^2.3.1" -webpack-dev-middleware@^1.0.11: +webpack-dev-middleware@^1.0.11, webpack-dev-middleware@^1.11.0: version "1.11.0" resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-1.11.0.tgz#09691d0973a30ad1f82ac73a12e2087f0a4754f9" dependencies: @@ -6456,16 +6389,6 @@ webpack-dev-middleware@^1.0.11: path-is-absolute "^1.0.0" range-parser "^1.0.3" -webpack-dev-middleware@^1.11.0: - version "1.12.0" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-1.12.0.tgz#d34efefb2edda7e1d3b5dbe07289513219651709" - dependencies: - memory-fs "~0.4.1" - mime "^1.3.4" - path-is-absolute "^1.0.0" - range-parser "^1.0.3" - time-stamp "^2.0.0" - webpack-dev-server@^2.6.1: version "2.7.1" resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-2.7.1.tgz#21580f5a08cd065c71144cf6f61c345bca59a8b8" @@ -6554,18 +6477,12 @@ which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" -which@^1.1.1, which@^1.2.1: +which@^1.1.1, which@^1.2.1, which@^1.2.9: version "1.2.12" resolved "https://registry.yarnpkg.com/which/-/which-1.2.12.tgz#de67b5e450269f194909ef23ece4ebe416fa1192" dependencies: isexe "^1.1.1" -which@^1.2.9: - version "1.3.0" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" - dependencies: - isexe "^2.0.0" - wide-align@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710" |
