diff options
40 files changed, 563 insertions, 31 deletions
diff --git a/app/assets/stylesheets/framework/modal.scss b/app/assets/stylesheets/framework/modal.scss index 46d40ea7aa5..ace46e32b18 100644 --- a/app/assets/stylesheets/framework/modal.scss +++ b/app/assets/stylesheets/framework/modal.scss @@ -101,3 +101,41 @@ body.modal-open { margin: 0; } } + +.issues-import-modal, +.issues-export-modal { + .modal-header { + justify-content: flex-start; + + .import-export-svg-container { + flex-grow: 1; + height: 56px; + padding: $gl-btn-padding $gl-btn-padding 0; + + > svg { + float: right; + height: 100%; + } + } + } + + .modal-body { + padding: 0; + + .modal-subheader { + justify-content: flex-start; + align-items: center; + border-bottom: 1px solid $modal-border-color; + padding: 14px; + } + + .modal-text { + padding: $gl-padding-24 $gl-padding; + min-height: $modal-body-height; + } + } + + .checkmark { + color: $green-400; + } +} diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index d92d81b2cb5..242977e8543 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -656,6 +656,7 @@ $border-color-settings: #e1e1e1; Modals */ $modal-body-height: 134px; +$modal-border-color: #e9ecef; $priority-label-empty-state-width: 114px; diff --git a/app/assets/stylesheets/pages/issues.scss b/app/assets/stylesheets/pages/issues.scss index bb6b6f84849..6c847fc0d53 100644 --- a/app/assets/stylesheets/pages/issues.scss +++ b/app/assets/stylesheets/pages/issues.scss @@ -155,6 +155,14 @@ ul.related-merge-requests > li { } } +.issues-nav-controls { + font-size: 0; + + .btn-group:empty { + display: none; + } +} + .issuable-email-modal-btn { padding: 0; color: $blue-600; diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index 0eea0cdd50f..c114e16edf8 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -7,12 +7,12 @@ module UploadsActions UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo favicon).freeze def create - link_to_file = UploadService.new(model, params[:file], uploader_class).execute + uploader = UploadService.new(model, params[:file], uploader_class).execute respond_to do |format| - if link_to_file + if uploader format.json do - render json: { link: link_to_file } + render json: { link: uploader.to_h } end else format.json do diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index 5ed46fc0545..21688e54481 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -10,7 +10,7 @@ class Projects::IssuesController < Projects::ApplicationController include SpammableActions def self.issue_except_actions - %i[index calendar new create bulk_update] + %i[index calendar new create bulk_update import_csv] end def self.set_issuables_index_only_actions @@ -37,6 +37,8 @@ class Projects::IssuesController < Projects::ApplicationController # Allow create a new branch and empty WIP merge request from current issue before_action :authorize_create_merge_request_from!, only: [:create_merge_request] + before_action :authorize_import_issues!, only: [:import_csv] + before_action :set_suggested_issues_feature_flags, only: [:new] respond_to :html @@ -175,6 +177,20 @@ class Projects::IssuesController < Projects::ApplicationController end end + def import_csv + return render_404 unless Feature.enabled?(:issues_import_csv) + + if uploader = UploadService.new(project, params[:file]).execute + ImportIssuesCsvWorker.perform_async(current_user.id, project.id, uploader.upload.id) + + flash[:notice] = _("Your issues are being imported. Once finished, you'll get a confirmation email.") + else + flash[:alert] = _("File upload error.") + end + + redirect_to project_issues_path(project) + end + protected # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/mailers/emails/issues.rb b/app/mailers/emails/issues.rb index 370e6d2f90b..654ae211310 100644 --- a/app/mailers/emails/issues.rb +++ b/app/mailers/emails/issues.rb @@ -77,6 +77,17 @@ module Emails mail_answer_thread(issue, issue_thread_options(updated_by_user.id, recipient.id, reason)) end + def import_issues_csv_email(user_id, project_id, results) + @user = User.find(user_id) + @project = Project.find(project_id) + @results = results + + mail(to: @user.notification_email, subject: subject('Imported issues')) do |format| + format.html { render layout: 'mailer' } + format.text { render layout: 'mailer' } + end + end + private def setup_issue_mail(issue_id, recipient_id) diff --git a/app/mailers/previews/notify_preview.rb b/app/mailers/previews/notify_preview.rb index 2ac4610967d..80e0a17c312 100644 --- a/app/mailers/previews/notify_preview.rb +++ b/app/mailers/previews/notify_preview.rb @@ -76,6 +76,10 @@ class NotifyPreview < ActionMailer::Preview Notify.changed_milestone_issue_email(user.id, issue.id, milestone, user.id) end + def import_issues_csv_email + Notify.import_issues_csv_email(user, project, { success: 3, errors: [5, 6, 7], valid_file: true }) + end + def closed_merge_request_email Notify.closed_merge_request_email(user.id, issue.id, user.id).message end diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3146f26bed5..d70417e710e 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -222,6 +222,8 @@ class ProjectPolicy < BasePolicy rule { owner | admin | guest | group_member }.prevent :request_access rule { ~request_access_enabled }.prevent :request_access + rule { can?(:developer_access) & can?(:create_issue) }.enable :import_issues + rule { can?(:developer_access) }.policy do enable :admin_merge_request enable :admin_milestone diff --git a/app/services/issues/import_csv_service.rb b/app/services/issues/import_csv_service.rb new file mode 100644 index 00000000000..ef08fafa7cc --- /dev/null +++ b/app/services/issues/import_csv_service.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Issues + class ImportCsvService + def initialize(user, project, csv_io) + @user = user + @project = project + @csv_io = csv_io + @results = { success: 0, error_lines: [], parse_error: false } + end + + def execute + process_csv + email_results_to_user + + @results + end + + private + + def process_csv + csv_data = @csv_io.open(&:read).force_encoding(Encoding::UTF_8) + + CSV.new(csv_data, col_sep: detect_col_sep(csv_data.lines.first), headers: true).each.with_index(2) do |row, line_no| + issue = Issues::CreateService.new(@project, @user, title: row[0], description: row[1]).execute + + if issue.persisted? + @results[:success] += 1 + else + @results[:error_lines].push(line_no) + end + end + rescue ArgumentError, CSV::MalformedCSVError + @results[:parse_error] = true + end + + def email_results_to_user + Notify.import_issues_csv_email(@user.id, @project.id, @results).deliver_now + end + + def detect_col_sep(header) + if header.include?(",") + "," + elsif header.include?(";") + ";" + elsif header.include?("\t") + "\t" + else + raise CSV::MalformedCSVError + end + end + end +end diff --git a/app/services/upload_service.rb b/app/services/upload_service.rb index 39909ee4f82..41ca95b3b6f 100644 --- a/app/services/upload_service.rb +++ b/app/services/upload_service.rb @@ -11,7 +11,7 @@ class UploadService uploader = @uploader_class.new(@model, nil, @uploader_context) uploader.store!(@file) - uploader.to_h + uploader end private diff --git a/app/views/notify/import_issues_csv_email.html.haml b/app/views/notify/import_issues_csv_email.html.haml new file mode 100644 index 00000000000..f30d2b5f078 --- /dev/null +++ b/app/views/notify/import_issues_csv_email.html.haml @@ -0,0 +1,18 @@ +- text_style = 'font-size:16px; text-align:center; line-height:30px;' + +%p{ style: text_style } + Your CSV import for project + %a{ href: project_url(@project), style: "color:#3777b0; text-decoration:none;" } + = @project.full_name + has been completed. + +%p{ style: text_style } + #{pluralize(@results[:success], 'issue')} imported. + +- if @results[:error_lines].present? + %p{ style: text_style } + Errors found on line #{'number'.pluralize(@results[:error_lines].size)}: #{@results[:error_lines].join(', ')}. Please check if these lines have an issue title. + +- if @results[:parse_error] + %p{ style: text_style } + Error parsing CSV file. Please make sure it has the correct format: a delimited text file that uses a comma to separate values. diff --git a/app/views/notify/import_issues_csv_email.text.erb b/app/views/notify/import_issues_csv_email.text.erb new file mode 100644 index 00000000000..1117f90714d --- /dev/null +++ b/app/views/notify/import_issues_csv_email.text.erb @@ -0,0 +1,11 @@ +Your CSV import for project <%= @project.full_name %> (<%= project_url(@project) %>) has been completed. + +<%= pluralize(@results[:success], 'issue') %> imported. + +<% if @results[:error_lines].present? %> +Errors found on line <%= 'number'.pluralize(@results[:error_lines].size) %>: <%= @results[:error_lines].join(', ') %>. Please check if these lines have an issue title. +<% end %> + +<% if @results[:parse_error] %> +Error parsing CSV file. Please make sure it has the correct format: a delimited text file that uses a comma to separate values. +<% end %> diff --git a/app/views/projects/cleanup/_show.html.haml b/app/views/projects/cleanup/_show.html.haml index cecc139b183..888be4ee282 100644 --- a/app/views/projects/cleanup/_show.html.haml +++ b/app/views/projects/cleanup/_show.html.haml @@ -24,6 +24,6 @@ = _("No file selected") = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true .form-text.text-muted - = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size } + = _("The maximum file size allowed is %{size}.") % { size: number_to_human_size(Gitlab::CurrentSettings.max_attachment_size.megabytes) } = f.submit _('Start cleanup'), class: 'btn btn-success' diff --git a/app/views/projects/issues/_import_export.svg b/app/views/projects/issues/_import_export.svg new file mode 100644 index 00000000000..53c35d12f57 --- /dev/null +++ b/app/views/projects/issues/_import_export.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 238 111" xmlns:xlink="http://www.w3.org/1999/xlink"><defs><rect id="4" width="82" rx="3" height="28" fill="#fff"/><path id="5" d="m68.926 12.09v-2.41c0-.665-.437-.888-.975-.507l-6.552 4.631c-.542.383-.539.998 0 1.379l6.552 4.631c.542.383.975.154.975-.507v-2.41h4.874c.668 0 1.2-.538 1.2-1.201v-2.406c0-.668-.537-1.201-1.2-1.201h-4.874" fill="#fc8a51"/><path id="6" d="m4 24h74v-20h-74v20m-4-21c0-1.655 1.338-2.996 2.991-2.996h76.02c1.652 0 2.991 1.35 2.991 2.996v22.01c0 1.655-1.338 2.996-2.991 2.996h-76.02c-1.652 0-2.991-1.35-2.991-2.996v-22.01"/><circle id="2" cx="16" cy="14" r="7"/><circle id="0" cx="16" cy="14" r="7"/><mask id="3" width="14" height="14" x="0" y="0" fill="#fff"><use xlink:href="#2"/></mask><mask id="1" width="14" height="14" x="0" y="0" fill="#fff"><use xlink:href="#0"/></mask></defs><g fill="none" fill-rule="evenodd"><rect width="98" height="111" fill="#fff" rx="6"/><path fill="#e5e5e5" fill-rule="nonzero" d="m4 6.01v98.99c0 1.11.897 2.01 2 2.01h85.998c1.105 0 2-.897 2-2.01v-98.99c0-1.11-.897-2.01-2-2.01h-85.998c-1.105 0-2 .897-2 2.01m-4 0c0-3.318 2.685-6.01 6-6.01h85.998c3.314 0 6 2.689 6 6.01v98.99c0 3.318-2.685 6.01-6 6.01h-85.998c-3.314 0-6-2.689-6-6.01v-98.99"/><rect width="76" height="85" x="11" y="12" fill="#f9f9f9" rx="3"/><g transform="translate(37 59)"><use xlink:href="#4"/><path fill="#e5e5e5" fill-rule="nonzero" d="m4 24h74v-20h-74v20m-4-21c0-1.655 1.338-2.996 2.991-2.996h76.02c1.652 0 2.991 1.35 2.991 2.996v22.01c0 1.655-1.338 2.996-2.991 2.996h-76.02c-1.652 0-2.991-1.35-2.991-2.996v-22.01"/><use fill="#fff" stroke="#6b4fbb" stroke-width="8" mask="url(#1)" xlink:href="#0"/><use xlink:href="#5"/></g><g transform="translate(140)"><path fill="#fff" d="m0 4h94v103h-94z"/><path fill="#e5e5e5" fill-rule="nonzero" d="m0 74v30.993c0 3.318 2.687 6.01 6 6.01h85.998c3.316 0 6-2.69 6-6.01v-98.99c0-3.318-2.687-6.01-6-6.01h-85.998c-3.316 0-6 2.69-6 6.01v.993h4v-.993c0-1.11.896-2.01 2-2.01h85.998c1.105 0 2 .897 2 2.01v98.99c0 1.11-.896 2.01-2 2.01h-85.998c-1.105 0-2-.897-2-2.01v-30.993h-4"/><g fill="#f9f9f9"><rect width="82" height="28" x="8" y="12" rx="3"/><rect width="82" height="28" x="8" y="43" rx="3"/></g></g><g fill-rule="nonzero" transform="translate(148 73)"><use fill="#e5e5e5" xlink:href="#6"/><path fill="#6b4fbb" d="m17 17c1.657 0 3-1.343 3-3 0-1.657-1.343-3-3-3-1.657 0-3 1.343-3 3 0 1.657 1.343 3 3 3m0 4c-3.866 0-7-3.134-7-7 0-3.866 3.134-7 7-7 3.866 0 7 3.134 7 7 0 3.866-3.134 7-7 7"/></g><g transform="translate(25 24)"><use xlink:href="#4"/><use fill="#e5e5e5" fill-rule="nonzero" xlink:href="#6"/><use fill="#fff" stroke="#6b4fbb" stroke-width="8" mask="url(#3)" xlink:href="#2"/><use xlink:href="#5"/></g><g transform="translate(107 10)"><use xlink:href="#4"/><use fill="#fc8a51" fill-opacity=".3" fill-rule="nonzero" xlink:href="#6"/><path fill="#6b4fbb" fill-rule="nonzero" d="m16 17c1.657 0 3-1.343 3-3 0-1.657-1.343-3-3-3-1.657 0-3 1.343-3 3 0 1.657 1.343 3 3 3m0 4c-3.866 0-7-3.134-7-7 0-3.866 3.134-7 7-7 3.866 0 7 3.134 7 7 0 3.866-3.134 7-7 7" id="7"/><use xlink:href="#5"/></g><g transform="translate(128 41)"><use xlink:href="#4"/><use fill="#fc8a51" fill-opacity=".3" fill-rule="nonzero" xlink:href="#6"/><use xlink:href="#7"/><path fill="#fc8a51" d="m66.926 12.09v-2.41c0-.665-.437-.888-.975-.507l-6.552 4.631c-.542.383-.539.998 0 1.379l6.552 4.631c.542.383.975.154.975-.507v-2.41h4.874c.668 0 1.2-.538 1.2-1.201v-2.406c0-.668-.537-1.201-1.2-1.201h-4.874"/></g></g></svg>
\ No newline at end of file diff --git a/app/views/projects/issues/_nav_btns.html.haml b/app/views/projects/issues/_nav_btns.html.haml index e4a0d4b8479..fd6559e37ba 100644 --- a/app/views/projects/issues/_nav_btns.html.haml +++ b/app/views/projects/issues/_nav_btns.html.haml @@ -1,11 +1,30 @@ -= render 'shared/issuable/feed_buttons' - -- if @can_bulk_update - = button_tag "Edit issues", class: "btn btn-default append-right-10 js-bulk-update-toggle" -- if show_new_issue_link?(@project) - = link_to "New issue", new_project_issue_path(@project, - issue: { assignee_id: finder.assignee.try(:id), - milestone_id: finder.milestones.first.try(:id) }), - class: "btn btn-success", - title: "New issue", - id: "new_issue_link" +- show_feed_buttons = local_assigns.fetch(:show_feed_buttons, true) +- show_import_button = local_assigns.fetch(:show_import_button, true) && Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, @project) +- show_export_button = local_assigns.fetch(:show_export_button, true) + +.nav-controls.issues-nav-controls + - if show_feed_buttons + = render 'shared/issuable/feed_buttons' + + .btn-group.append-right-10< + - if show_export_button + = render_if_exists 'projects/issues/export_csv/button' + + - if show_import_button + = render 'projects/issues/import_csv/button' + + - if @can_bulk_update + = button_tag _("Edit issues"), class: "btn btn-default append-right-10 js-bulk-update-toggle" + - if show_new_issue_link?(@project) + = link_to _("New issue"), new_project_issue_path(@project, + issue: { assignee_id: finder.assignee.try(:id), + milestone_id: finder.milestones.first.try(:id) }), + class: "btn btn-success", + title: _("New issue"), + id: "new_issue_link" + +- if show_export_button + = render_if_exists 'projects/issues/export_csv/modal' + +- if show_import_button + = render 'projects/issues/import_csv/modal' diff --git a/app/views/projects/issues/import_csv/_button.html.haml b/app/views/projects/issues/import_csv/_button.html.haml new file mode 100644 index 00000000000..acc2c50294f --- /dev/null +++ b/app/views/projects/issues/import_csv/_button.html.haml @@ -0,0 +1,9 @@ +- type = local_assigns.fetch(:type, :icon) + +%button.csv-import-button.btn{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon), + data: { toggle: 'modal', target: '.issues-import-modal' } } + - if type == :icon + = sprite_icon('upload') + - else + = _('Import CSV') + diff --git a/app/views/projects/issues/import_csv/_modal.html.haml b/app/views/projects/issues/import_csv/_modal.html.haml new file mode 100644 index 00000000000..18768307341 --- /dev/null +++ b/app/views/projects/issues/import_csv/_modal.html.haml @@ -0,0 +1,24 @@ +.issues-import-modal.modal + .modal-dialog + .modal-content + = form_tag import_csv_namespace_project_issues_path, multipart: true do + .modal-header + %h3 + = _('Import issues') + .import-export-svg-container + = render 'projects/issues/import_export.svg' + %a.close{ href: '#', 'data-dismiss' => 'modal' } × + .modal-body + .modal-text + %p + = _("Your issues will be imported in the background. Once finished, you'll get a confirmation email.") + .form-group + = label_tag :file, _('Upload CSV file'), class: 'label-bold' + %div + = file_field_tag :file, accept: '.csv,text/csv', required: true + %p.text-secondary + = _('It must have a header row and at least two columns: the first column is the issue title and the second column is the issue description. The separator is automatically detected.') + = _('The maximum file size allowed is %{size}.') % { size: number_to_human_size(Gitlab::CurrentSettings.max_attachment_size.megabytes) } + .modal-footer + %button{ type: 'submit', class: 'btn btn-success', title: _('Import issues') } + = _('Import issues') diff --git a/app/views/projects/issues/index.html.haml b/app/views/projects/issues/index.html.haml index 1e7737aeb97..39e9e9171cf 100644 --- a/app/views/projects/issues/index.html.haml +++ b/app/views/projects/issues/index.html.haml @@ -11,8 +11,7 @@ %div{ class: (container_class) } .top-area = render 'shared/issuable/nav', type: :issues - .nav-controls - = render "projects/issues/nav_btns" + = render "projects/issues/nav_btns" = render 'shared/issuable/search_bar', type: :issues - if @can_bulk_update @@ -23,4 +22,4 @@ - if new_issue_email = render 'projects/issuable_by_email', email: new_issue_email, issuable_type: 'issue' - else - = render 'shared/empty_states/issues', button_path: new_project_issue_path(@project) + = render 'shared/empty_states/issues', button_path: new_project_issue_path(@project), show_import_button: true diff --git a/app/views/shared/empty_states/_issues.html.haml b/app/views/shared/empty_states/_issues.html.haml index 0ddc56dc6c3..0434860dec4 100644 --- a/app/views/shared/empty_states/_issues.html.haml +++ b/app/views/shared/empty_states/_issues.html.haml @@ -1,5 +1,6 @@ - button_path = local_assigns.fetch(:button_path, false) - project_select_button = local_assigns.fetch(:project_select_button, false) +- show_import_button = local_assigns.fetch(:show_import_button, false) && Feature.enabled?(:issues_import_csv) && can?(current_user, :import_issues, @project) - has_button = button_path || project_select_button .row.empty-state @@ -21,12 +22,20 @@ - if has_button .text-center - if project_select_button - = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue', type: :issues, with_feature_enabled: 'issues' + = render 'shared/new_project_item_select', path: 'issues/new', label: _('New issue'), type: :issues, with_feature_enabled: 'issues' - else - = link_to 'New issue', button_path, class: 'btn btn-success', title: 'New issue', id: 'new_issue_link' + = link_to _('New issue'), button_path, class: 'btn btn-success', title: _('New issue'), id: 'new_issue_link' + + - if show_import_button + = render 'projects/issues/import_csv/button', type: :text + - else %h4.text-center= _("There are no issues to show") %p = _("The Issue Tracker is the place to add things that need to be improved or solved in a project. You can register or sign in to create issues for this project.") .text-center = link_to _('Register / Sign In'), new_user_session_path, class: 'btn btn-success' + +- if show_import_button + = render 'projects/issues/import_csv/modal' + diff --git a/app/views/shared/issuable/_feed_buttons.html.haml b/app/views/shared/issuable/_feed_buttons.html.haml index d4834090413..83f60fa6fe2 100644 --- a/app/views/shared/issuable/_feed_buttons.html.haml +++ b/app/views/shared/issuable/_feed_buttons.html.haml @@ -1,4 +1,4 @@ -= link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: 'Subscribe to RSS feed' do += link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to RSS feed') do = icon('rss') -= link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: 'Subscribe to calendar' do += link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do = custom_icon('icon_calendar') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d3cf21db335..223ddc80c88 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -140,3 +140,4 @@ - detect_repository_languages - repository_cleanup - delete_stored_files +- import_issues_csv diff --git a/app/workers/import_issues_csv_worker.rb b/app/workers/import_issues_csv_worker.rb new file mode 100644 index 00000000000..d44fdfec8ae --- /dev/null +++ b/app/workers/import_issues_csv_worker.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class ImportIssuesCsvWorker + include ApplicationWorker + + sidekiq_retries_exhausted do |job| + Upload.find(job['args'][2]).destroy + end + + def perform(current_user_id, project_id, upload_id) + @user = User.find(current_user_id) + @project = Project.find(project_id) + @upload = Upload.find(upload_id) + + importer = Issues::ImportCsvService.new(@user, @project, @upload.build_uploader) + importer.execute + + @upload.destroy + end +end diff --git a/changelogs/unreleased/49231-import-issues-csv.yml b/changelogs/unreleased/49231-import-issues-csv.yml new file mode 100644 index 00000000000..c10bd8143b2 --- /dev/null +++ b/changelogs/unreleased/49231-import-issues-csv.yml @@ -0,0 +1,5 @@ +--- +title: Add importing of issues from CSV file +merge_request: 23532 +author: +type: added diff --git a/config/routes/project.rb b/config/routes/project.rb index f50bf5ab76f..cf5a57300cf 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -361,6 +361,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end collection do post :bulk_update + post :import_csv end end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 3ee32678f34..3e8c218052d 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -85,3 +85,4 @@ - [repository_cleanup, 1] - [delete_stored_files, 1] - [remote_mirror_notification, 2] + - [import_issues_csv, 2] diff --git a/doc/user/project/issues/csv_import.md b/doc/user/project/issues/csv_import.md new file mode 100644 index 00000000000..001e0d303e9 --- /dev/null +++ b/doc/user/project/issues/csv_import.md @@ -0,0 +1,45 @@ +# Importing Issues from CSV + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/23532) in GitLab 11.7. + +Issues can be imported by uploading a CSV file. The file will be processed in the background and a notification email +will be sent to you once the import is completed. + +> **Note:** A permission level of `Developer` or higher is required to import issues. + +## CSV File Format + +### Header row + +CSV files must contain a header row with at least two columns: `title` and `description`, in that order. + +### Column separator + +The column separator is automatically detected from the header row. + +Supported separator characters are: commas (`,`), semicolons (`;`), and tabs (`\t`). + +### Row separator + +Lines ending in either `CRLF` or `LF` are supported. + +### Quote character + +The double-quote (`"`) character is used to quote fields so you can use the column separator within a field. To insert +a double-quote (`"`) within a quoted field, use two double-quote characters in succession, i.e. `""`. + +### Data rows + +After the header row, succeeding rows must follow the same column order. The issue title is required while the +description is optional. + +The user uploading the CSV file will be set as the author of the imported issues. + +## Sample Data + +```csv +title,description +My Issue Title,My Issue Description +Another Title,"A description, with a comma" +"One More Title","One More Description" +``` diff --git a/doc/user/project/issues/img/import_csv_button.png b/doc/user/project/issues/img/import_csv_button.png Binary files differnew file mode 100644 index 00000000000..ab100a95750 --- /dev/null +++ b/doc/user/project/issues/img/import_csv_button.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 200b3a642a1..40a1f60c4ab 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -142,6 +142,15 @@ to find out more about this feature. With [GitLab Starter](https://about.gitlab.com/pricing/), you can also create various boards per project with [Multiple Issue Boards](https://docs.gitlab.com/ee/user/project/issue_board.html#multiple-issue-boards). +### Import Issues from CSV + +From the project-level issues list, you can find the import button near the "Edit issues" button in the upper-right +side. + + + +Learn more about [importing issues from CSV](csv_import.md) + ### External Issue Tracker Alternatively to GitLab's built-in Issue Tracker, you can also use an [external diff --git a/lib/api/projects.rb b/lib/api/projects.rb index f5d21d8923f..9f3a1699146 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -475,7 +475,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute.to_h end desc 'Get the users list of a project' do diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index a826519b2dd..3323ce60158 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -23,8 +23,8 @@ module Gitlab content_type: attachment.content_type } - link = UploadService.new(project, file).execute - attachments << link if link + uploader = UploadService.new(project, file).execute + attachments << uploader.to_h if uploader ensure tmp.close! end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 474e9d45566..e232198150a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -40,7 +40,7 @@ module Gitlab def add_upload(upload) uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys - UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h end def copy_project_uploads diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 6e38425034f..7db75b5cfef 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2713,6 +2713,9 @@ msgstr "" msgid "Edit identity for %{user_name}" msgstr "" +msgid "Edit issues" +msgstr "" + msgid "Email" msgstr "" @@ -3103,6 +3106,9 @@ msgstr "" msgid "File templates" msgstr "" +msgid "File upload error." +msgstr "" + msgid "Files" msgstr "" @@ -3615,6 +3621,9 @@ msgstr "" msgid "Import" msgstr "" +msgid "Import CSV" +msgstr "" + msgid "Import Projects from Gitea" msgstr "" @@ -3633,6 +3642,9 @@ msgstr "" msgid "Import in progress" msgstr "" +msgid "Import issues" +msgstr "" + msgid "Import multiple repositories by uploading a manifest file." msgstr "" @@ -3771,6 +3783,9 @@ msgstr "" msgid "Issues, merge requests, pushes and comments." msgstr "" +msgid "It must have a header row and at least two columns: the first column is the issue title and the second column is the issue description. The separator is automatically detected." +msgstr "" + msgid "It's you" msgstr "" @@ -6506,6 +6521,12 @@ msgstr "" msgid "Subscribe at project level" msgstr "" +msgid "Subscribe to RSS feed" +msgstr "" + +msgid "Subscribe to calendar" +msgstr "" + msgid "Subscribed" msgstr "" @@ -6665,7 +6686,7 @@ msgstr "" msgid "The issue stage shows the time it takes from creating an issue to assigning the issue to a milestone, or add the issue to a list on your Issue Board. Begin creating issues to see data for this stage." msgstr "" -msgid "The maximum file size allowed is %{max_attachment_size}mb" +msgid "The maximum file size allowed is %{size}." msgstr "" msgid "The maximum file size allowed is 200KB." @@ -7362,6 +7383,9 @@ msgstr "" msgid "Upload <code>GoogleCodeProjectHosting.json</code> here:" msgstr "" +msgid "Upload CSV file" +msgstr "" + msgid "Upload New File" msgstr "" @@ -7911,6 +7935,12 @@ msgstr "" msgid "Your groups" msgstr "" +msgid "Your issues are being imported. Once finished, you'll get a confirmation email." +msgstr "" + +msgid "Your issues will be imported in the background. Once finished, you'll get a confirmation email." +msgstr "" + msgid "Your name" msgstr "" diff --git a/spec/controllers/projects/issues_controller_spec.rb b/spec/controllers/projects/issues_controller_spec.rb index a239ac16c0d..df21dc7bc85 100644 --- a/spec/controllers/projects/issues_controller_spec.rb +++ b/spec/controllers/projects/issues_controller_spec.rb @@ -1026,6 +1026,72 @@ describe Projects::IssuesController do end end + describe 'POST #import_csv' do + let(:project) { create(:project, :public) } + let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') } + + context 'feature disabled' do + it 'returns 404' do + sign_in(user) + project.add_maintainer(user) + + stub_feature_flags(issues_import_csv: false) + + import_csv + + expect(response).to have_gitlab_http_status :not_found + end + end + + context 'unauthorized' do + it 'returns 404 for guests' do + sign_out(:user) + + import_csv + + expect(response).to have_gitlab_http_status :not_found + end + + it 'returns 404 for project members with reporter role' do + sign_in(user) + project.add_reporter(user) + + import_csv + + expect(response).to have_gitlab_http_status :not_found + end + end + + context 'authorized' do + before do + sign_in(user) + project.add_developer(user) + end + + it "returns 302 for project members with developer role" do + import_csv + + expect(flash[:notice]).to include('Your issues are being imported') + expect(response).to redirect_to(project_issues_path(project)) + end + + it "shows error when upload fails" do + allow_any_instance_of(UploadService).to receive(:execute).and_return(nil) + + import_csv + + expect(flash[:alert]).to include('File upload error.') + expect(response).to redirect_to(project_issues_path(project)) + end + end + + def import_csv + post :import_csv, namespace_id: project.namespace.to_param, + project_id: project.to_param, + file: file + end + end + describe 'GET #discussions' do let!(:discussion) { create(:discussion_note_on_issue, noteable: issue, project: issue.project) } context 'when authenticated' do diff --git a/spec/fixtures/csv_comma.csv b/spec/fixtures/csv_comma.csv new file mode 100644 index 00000000000..e477a27d243 --- /dev/null +++ b/spec/fixtures/csv_comma.csv @@ -0,0 +1,4 @@ +title,description +Issue in 中文,Test description +"Hello","World" +"Title with quote""",Description diff --git a/spec/fixtures/csv_semicolon.csv b/spec/fixtures/csv_semicolon.csv new file mode 100644 index 00000000000..679797489e2 --- /dev/null +++ b/spec/fixtures/csv_semicolon.csv @@ -0,0 +1,5 @@ +title;description
+Issue in 中文;Test description
+Title with, comma;"Description"
+
+"Hello";"World"
diff --git a/spec/fixtures/csv_tab.csv b/spec/fixtures/csv_tab.csv new file mode 100644 index 00000000000..f801794ea9c --- /dev/null +++ b/spec/fixtures/csv_tab.csv @@ -0,0 +1,4 @@ +title description +Issue in 中文 Test description + "Error Row" +"Hello" "World" diff --git a/spec/mailers/emails/issues_spec.rb b/spec/mailers/emails/issues_spec.rb new file mode 100644 index 00000000000..09253cf8003 --- /dev/null +++ b/spec/mailers/emails/issues_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'email_spec' + +describe Emails::Issues do + include EmailSpec::Matchers + + describe "#import_issues_csv_email" do + let(:user) { create(:user) } + let(:project) { create(:project) } + + subject { Notify.import_issues_csv_email(user.id, project.id, @results) } + + it "shows number of successful issues imported" do + @results = { success: 165, error_lines: [], parse_error: false } + + expect(subject).to have_body_text "165 issues imported" + end + + it "shows error when file is invalid" do + @results = { success: 0, error_lines: [], parse_error: true } + + expect(subject).to have_body_text "Error parsing CSV" + end + + it "shows line numbers with errors" do + @results = { success: 0, error_lines: [23, 34, 58], parse_error: false } + + expect(subject).to have_body_text "23, 34, 58" + end + end +end diff --git a/spec/services/issues/import_csv_service_spec.rb b/spec/services/issues/import_csv_service_spec.rb new file mode 100644 index 00000000000..516a1137319 --- /dev/null +++ b/spec/services/issues/import_csv_service_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Issues::ImportCsvService do + let(:project) { create(:project) } + let(:user) { create(:user) } + + subject do + uploader = FileUploader.new(project) + uploader.store!(file) + + described_class.new(user, project, uploader).execute + end + + describe '#execute' do + context 'invalid file' do + let(:file) { fixture_file_upload('spec/fixtures/banana_sample.gif') } + + it 'returns invalid file error' do + expect_any_instance_of(Notify).to receive(:import_issues_csv_email) + + expect(subject[:success]).to eq(0) + expect(subject[:parse_error]).to eq(true) + end + end + + context 'comma delimited file' do + let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') } + + it 'imports CSV without errors' do + expect_any_instance_of(Notify).to receive(:import_issues_csv_email) + + expect(subject[:success]).to eq(3) + expect(subject[:error_lines]).to eq([]) + expect(subject[:parse_error]).to eq(false) + end + end + + context 'tab delimited file with error row' do + let(:file) { fixture_file_upload('spec/fixtures/csv_tab.csv') } + + it 'imports CSV with some error rows' do + expect_any_instance_of(Notify).to receive(:import_issues_csv_email) + + expect(subject[:success]).to eq(2) + expect(subject[:error_lines]).to eq([3]) + expect(subject[:parse_error]).to eq(false) + end + end + + context 'semicolon delimited file with CRLF' do + let(:file) { fixture_file_upload('spec/fixtures/csv_semicolon.csv') } + + it 'imports CSV with a blank row' do + expect_any_instance_of(Notify).to receive(:import_issues_csv_email) + + expect(subject[:success]).to eq(3) + expect(subject[:error_lines]).to eq([4]) + expect(subject[:parse_error]).to eq(false) + end + end + end +end diff --git a/spec/services/upload_service_spec.rb b/spec/services/upload_service_spec.rb index 9b232a52efa..4a809d5bf18 100644 --- a/spec/services/upload_service_spec.rb +++ b/spec/services/upload_service_spec.rb @@ -63,11 +63,11 @@ describe UploadService do @link_to_file = upload_file(@project, txt) end - it { expect(@link_to_file).to eq(nil) } + it { expect(@link_to_file).to eq({}) } end end def upload_file(project, file) - described_class.new(project, file, FileUploader).execute + described_class.new(project, file, FileUploader).execute.to_h end end diff --git a/spec/workers/import_issues_csv_worker_spec.rb b/spec/workers/import_issues_csv_worker_spec.rb new file mode 100644 index 00000000000..89370c4890d --- /dev/null +++ b/spec/workers/import_issues_csv_worker_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe ImportIssuesCsvWorker do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:upload) { create(:upload) } + + let(:worker) { described_class.new } + + describe '#perform' do + it 'calls #execute on Issues::ImportCsvService and destroys upload' do + expect_any_instance_of(Issues::ImportCsvService).to receive(:execute).and_return({ success: 5, errors: [], valid_file: true }) + + worker.perform(user.id, project.id, upload.id) + + expect { upload.reload }.to raise_error ActiveRecord::RecordNotFound + end + end +end |
