Merge branch '22348-gitea-importer' into 'master'

It adds a brand new importer for Gitea!

This is a continuation of !6945 started by @bkc.

Gitea aims to be 100% GitHub-compatible but there's a few differences:

- Gitea is not an OAuth provider (yet): https://github.com/go-gitea/gitea/issues/27
  - This means we cannot map Gitea users given an assignee ID => assignees are not set on imported issues and merge requests
- No releases API for now: https://github.com/go-gitea/gitea/issues/330
- API version is `v1` (GitHub is `v3`)
- The IID field for milestones is `id` compared to `number` in GitHub.
- Issues, PRs, milestones, labels don't have a `url` field (the importer now fallback to `''` in that case)

**Known issues:**

- Comments are not imported because comments JSON always have a blank `html_url`/`issue_url`/`pull_request_url`, so the IID cannot be extracted and the issuable cannot be found... :( This is tracked in https://github.com/go-gitea/gitea/issues/401, and solved by https://github.com/gogits/gogs/pull/3624 but this needs to be submitted / merged in Gitea.

This is noted in the documentation.

## Are there points in the code the reviewer needs to double check?

1. I've made `Import::GiteaController` inherit from `Import::GithubController` since both controllers should be identical in the long-term and their current differences are small.
1. I've added a base `IssuableFormatter` class from which `IssueFormatter` & `PullRequestFormatter` inherit
1. I've added shared examples for GitHub/Gitea importer classes
1. I've made `Gitlab::ImportSources` more robust and tested! :christmas_tree: 
1. I've added routing specs for import routes! :christmas_tree: 

Closes #22348

See merge request !8116

12 files changed, 225 insertions, 112 deletions

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 4df80195ae1..bb47e2a8bf7 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -25,7 +25,7 @@ class ApplicationController < ActionController::Base
   protect_from_forgery with: :exception
 
   helper_method :can?, :current_application_settings
-  helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
+  helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
 
   rescue_from Encoding::CompatibilityError do |exception|
     log_exception(exception)
@@ -245,6 +245,10 @@ class ApplicationController < ActionController::Base
     current_application_settings.import_sources.include?('github')
   end
 
+  def gitea_import_enabled?
+    current_application_settings.import_sources.include?('gitea')
+  end
+
   def github_import_configured?
     Gitlab::OAuth::Provider.enabled?(:github)
   end
diff --git a/app/controllers/import/gitea_controller.rb b/app/controllers/import/gitea_controller.rb
new file mode 100644
index 00000000000..fbd851c64a7
--- /dev/null
+++ b/app/controllers/import/gitea_controller.rb
@@ -0,0 +1,45 @@
+class Import::GiteaController < Import::GithubController
+  def new
+    if session[access_token_key].present? && session[host_key].present?
+      redirect_to status_import_url
+    end
+  end
+
+  def personal_access_token
+    session[host_key] = params[host_key]
+    super
+  end
+
+  def status
+    @gitea_host_url = session[host_key]
+    super
+  end
+
+  private
+
+  def host_key
+    :"#{provider}_host_url"
+  end
+
+  # Overriden methods
+  def provider
+    :gitea
+  end
+
+  # Gitea is not yet an OAuth provider
+  # See https://github.com/go-gitea/gitea/issues/27
+  def logged_in_with_provider?
+    false
+  end
+
+  def provider_auth
+    if session[access_token_key].blank? || session[host_key].blank?
+      redirect_to new_import_gitea_url,
+        alert: 'You need to specify both an Access Token and a Host URL.'
+    end
+  end
+
+  def client_options
+    { host: session[host_key], api_version: 'v1' }
+  end
+end
diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb
index ee7d498c59c..53a5981e564 100644
--- a/app/controllers/import/github_controller.rb
+++ b/app/controllers/import/github_controller.rb
@@ -1,39 +1,37 @@
 class Import::GithubController < Import::BaseController
-  before_action :verify_github_import_enabled
-  before_action :github_auth, only: [:status, :jobs, :create]
+  before_action :verify_import_enabled
+  before_action :provider_auth, only: [:status, :jobs, :create]
 
-  rescue_from Octokit::Unauthorized, with: :github_unauthorized
-
-  helper_method :logged_in_with_github?
+  rescue_from Octokit::Unauthorized, with: :provider_unauthorized
 
   def new
-    if logged_in_with_github?
-      go_to_github_for_permissions
-    elsif session[:github_access_token]
-      redirect_to status_import_github_url
+    if logged_in_with_provider?
+      go_to_provider_for_permissions
+    elsif session[access_token_key]
+      redirect_to status_import_url
     end
   end
 
   def callback
-    session[:github_access_token] = client.get_token(params[:code])
-    redirect_to status_import_github_url
+    session[access_token_key] = client.get_token(params[:code])
+    redirect_to status_import_url
   end
 
   def personal_access_token
-    session[:github_access_token] = params[:personal_access_token]
-    redirect_to status_import_github_url
+    session[access_token_key] = params[:personal_access_token]
+    redirect_to status_import_url
   end
 
   def status
     @repos = client.repos
-    @already_added_projects = current_user.created_projects.where(import_type: "github")
+    @already_added_projects = current_user.created_projects.where(import_type: provider)
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
-    @repos.reject!{ |repo| already_added_projects_names.include? repo.full_name }
+    @repos.reject! { |repo| already_added_projects_names.include? repo.full_name }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: "github").to_json(only: [:id, :import_status])
+    jobs = current_user.created_projects.where(import_type: provider).to_json(only: [:id, :import_status])
     render json: jobs
   end
 
@@ -44,8 +42,8 @@ class Import::GithubController < Import::BaseController
     namespace_path = params[:target_namespace].presence || current_user.namespace_path
     @target_namespace = find_or_create_namespace(namespace_path, current_user.namespace_path)
 
-    if current_user.can?(:create_projects, @target_namespace)
-      @project = Gitlab::GithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params).execute
+    if can?(current_user, :create_projects, @target_namespace)
+      @project = Gitlab::GithubImport::ProjectCreator.new(repo, @project_name, @target_namespace, current_user, access_params, type: provider).execute
     else
       render 'unauthorized'
     end
@@ -54,34 +52,63 @@ class Import::GithubController < Import::BaseController
   private
 
   def client
-    @client ||= Gitlab::GithubImport::Client.new(session[:github_access_token])
+    @client ||= Gitlab::GithubImport::Client.new(session[access_token_key], client_options)
   end
 
-  def verify_github_import_enabled
-    render_404 unless github_import_enabled?
+  def verify_import_enabled
+    render_404 unless import_enabled?
   end
 
-  def github_auth
-    if session[:github_access_token].blank?
-      go_to_github_for_permissions
-    end
+  def go_to_provider_for_permissions
+    redirect_to client.authorize_url(callback_import_url)
   end
 
-  def go_to_github_for_permissions
-    redirect_to client.authorize_url(callback_import_github_url)
+  def import_enabled?
+    __send__("#{provider}_import_enabled?")
   end
 
-  def github_unauthorized
-    session[:github_access_token] = nil
-    redirect_to new_import_github_url,
-      alert: 'Access denied to your GitHub account.'
+  def new_import_url
+    public_send("new_import_#{provider}_url")
   end
 
-  def logged_in_with_github?
-    current_user.identities.exists?(provider: 'github')
+  def status_import_url
+    public_send("status_import_#{provider}_url")
+  end
+
+  def callback_import_url
+    public_send("callback_import_#{provider}_url")
+  end
+
+  def provider_unauthorized
+    session[access_token_key] = nil
+    redirect_to new_import_url,
+      alert: "Access denied to your #{Gitlab::ImportSources.title(provider.to_s)} account."
+  end
+
+  def access_token_key
+    :"#{provider}_access_token"
   end
 
   def access_params
-    { github_access_token: session[:github_access_token] }
+    { github_access_token: session[access_token_key] }
+  end
+
+  # The following methods are overriden in subclasses
+  def provider
+    :github
+  end
+
+  def logged_in_with_provider?
+    current_user.identities.exists?(provider: provider)
+  end
+
+  def provider_auth
+    if session[access_token_key].blank?
+      go_to_provider_for_permissions
+    end
+  end
+
+  def client_options
+    {}
   end
 end
diff --git a/app/helpers/import_helper.rb b/app/helpers/import_helper.rb
index 021d2b14718..a0642a1894b 100644
--- a/app/helpers/import_helper.rb
+++ b/app/helpers/import_helper.rb
@@ -4,8 +4,10 @@ module ImportHelper
     "#{namespace}/#{name}"
   end
 
-  def github_project_link(path_with_namespace)
-    link_to path_with_namespace, github_project_url(path_with_namespace), target: '_blank'
+  def provider_project_link(provider, path_with_namespace)
+    url = __send__("#{provider}_project_url", path_with_namespace)
+
+    link_to path_with_namespace, url, target: '_blank'
   end
 
   private
@@ -20,4 +22,8 @@ module ImportHelper
     provider = Gitlab.config.omniauth.providers.find { |p| p.name == 'github' }
     @github_url = provider.fetch('url', 'https://github.com') if provider
   end
+
+  def gitea_project_url(path_with_namespace)
+    "#{@gitea_host_url.sub(%r{/+\z}, '')}/#{path_with_namespace}"
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index 009843d6aed..72d3da64f2d 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -533,6 +533,10 @@ class Project < ActiveRecord::Base
     import_type == 'gitlab_project'
   end
 
+  def gitea_import?
+    import_type == 'gitea'
+  end
+
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
       projects_limit = creator.projects_limit
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index d7221fe993c..cd230528743 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -4,15 +4,6 @@ module Projects
 
     class Error < StandardError; end
 
-    ALLOWED_TYPES = [
-      'bitbucket',
-      'fogbugz',
-      'gitlab',
-      'github',
-      'google_code',
-      'gitlab_project'
-    ]
-
     def execute
       add_repository_to_project unless project.gitlab_project_import?
 
@@ -64,14 +55,11 @@ module Projects
     end
 
     def has_importer?
-      ALLOWED_TYPES.include?(project.import_type)
+      Gitlab::ImportSources.importer_names.include?(project.import_type)
     end
 
     def importer
-      return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import?
-
-      class_name = "Gitlab::#{project.import_type.camelize}Import::Importer"
-      class_name.constantize.new(project)
+      Gitlab::ImportSources.importer(project.import_type).new(project)
     end
 
     def unknown_url?
diff --git a/app/views/import/_githubish_status.html.haml b/app/views/import/_githubish_status.html.haml
new file mode 100644
index 00000000000..f12f9482a51
--- /dev/null
+++ b/app/views/import/_githubish_status.html.haml
@@ -0,0 +1,61 @@
+- provider = local_assigns.fetch(:provider)
+- provider_title = Gitlab::ImportSources.title(provider)
+
+%p.light
+  Select projects you want to import.
+%hr
+%p
+  = button_tag class: "btn btn-import btn-success js-import-all" do
+    Import all projects
+    = icon("spinner spin", class: "loading-icon")
+
+.table-responsive
+  %table.table.import-jobs
+    %colgroup.import-jobs-from-col
+    %colgroup.import-jobs-to-col
+    %colgroup.import-jobs-status-col
+    %thead
+      %tr
+        %th= "From #{provider_title}"
+        %th To GitLab
+        %th Status
+    %tbody
+      - @already_added_projects.each do |project|
+        %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+          %td
+            = provider_project_link(provider, project.import_source)
+          %td
+            = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
+          %td.job-status
+            - if project.import_status == 'finished'
+              %span
+                %i.fa.fa-check
+                done
+            - elsif project.import_status == 'started'
+              %i.fa.fa-spinner.fa-spin
+              started
+            - else
+              = project.human_import_status_name
+
+      - @repos.each do |repo|
+        %tr{id: "repo_#{repo.id}"}
+          %td
+            = provider_project_link(provider, repo.full_name)
+          %td.import-target
+            %fieldset.row
+            .input-group
+              .project-path.input-group-btn
+                - if current_user.can_select_namespace?
+                  - selected = params[:namespace_id] || :current_user
+                  - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner.login, path: repo.owner.login) } : {}
+                  = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
+                - else
+                  = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
+              %span.input-group-addon /
+              = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
+          %td.import-actions.job-status
+            = button_tag class: "btn btn-import js-add-to-import" do
+              Import
+              = icon("spinner spin", class: "loading-icon")
+
+.js-importer-status{ data: { jobs_import_path: "#{url_for([:jobs, :import, provider])}", import_path: "#{url_for([:import, provider])}" } }
diff --git a/app/views/import/gitea/new.html.haml b/app/views/import/gitea/new.html.haml
new file mode 100644
index 00000000000..02a116f996b
--- /dev/null
+++ b/app/views/import/gitea/new.html.haml
@@ -0,0 +1,23 @@
+- page_title "Gitea Import"
+- header_title "Projects", root_path
+
+%h3.page-title
+  = custom_icon('go_logo')
+  Import Projects from Gitea
+
+%p
+  To get started, please enter your Gitea Host URL and a
+  = succeed '.' do
+    = link_to 'Personal Access Token', 'https://github.com/gogits/go-gogs-client/wiki#access-token'
+
+= form_tag personal_access_token_import_gitea_path, class: 'form-horizontal' do
+  .form-group
+    = label_tag :gitea_host_url, 'Gitea Host URL', class: 'control-label'
+    .col-sm-4
+      = text_field_tag :gitea_host_url, nil, placeholder: 'https://try.gitea.io', class: 'form-control'
+  .form-group
+    = label_tag :personal_access_token, 'Personal Access Token', class: 'control-label'
+    .col-sm-4
+      = text_field_tag :personal_access_token, nil, class: 'form-control'
+  .form-actions
+    = submit_tag 'List Your Gitea Repositories', class: 'btn btn-create'
diff --git a/app/views/import/gitea/status.html.haml b/app/views/import/gitea/status.html.haml
new file mode 100644
index 00000000000..589ca27e45d
--- /dev/null
+++ b/app/views/import/gitea/status.html.haml
@@ -0,0 +1,7 @@
+- page_title "Gitea Import"
+- header_title "Projects", root_path
+%h3.page-title
+  = custom_icon('go_logo')
+  Import Projects from Gitea
+
+= render 'import/githubish_status', provider: 'gitea'
diff --git a/app/views/import/github/status.html.haml b/app/views/import/github/status.html.haml
index 4c721d40b55..0fe578a0036 100644
--- a/app/views/import/github/status.html.haml
+++ b/app/views/import/github/status.html.haml
@@ -1,64 +1,6 @@
-- page_title "GitHub import"
+- page_title "GitHub Import"
 - header_title "Projects", root_path
 %h3.page-title
-  %i.fa.fa-github
-  Import projects from GitHub
+  = icon 'github', text: 'Import Projects from GitHub'
 
-%p.light
-  Select projects you want to import.
-%hr
-%p
-  = button_tag class: "btn btn-import btn-success js-import-all" do
-    Import all projects
-    = icon("spinner spin", class: "loading-icon")
-
-.table-responsive
-  %table.table.import-jobs
-    %colgroup.import-jobs-from-col
-    %colgroup.import-jobs-to-col
-    %colgroup.import-jobs-status-col
-    %thead
-      %tr
-        %th From GitHub
-        %th To GitLab
-        %th Status
-    %tbody
-      - @already_added_projects.each do |project|
-        %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
-          %td
-            = github_project_link(project.import_source)
-          %td
-            = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
-          %td.job-status
-            - if project.import_status == 'finished'
-              %span
-                %i.fa.fa-check
-                done
-            - elsif project.import_status == 'started'
-              %i.fa.fa-spinner.fa-spin
-              started
-            - else
-              = project.human_import_status_name
-
-      - @repos.each do |repo|
-        %tr{id: "repo_#{repo.id}"}
-          %td
-            = github_project_link(repo.full_name)
-          %td.import-target
-            %fieldset.row
-            .input-group
-              .project-path.input-group-btn
-                - if current_user.can_select_namespace?
-                  - selected = params[:namespace_id] || :current_user
-                  - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner.login, path: repo.owner.login) } : {}
-                  = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
-                - else
-                  = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
-              %span.input-group-addon /
-              = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
-          %td.import-actions.job-status
-            = button_tag class: "btn btn-import js-add-to-import" do
-              Import
-              = icon("spinner spin", class: "loading-icon")
-
-.js-importer-status{ data: { jobs_import_path: "#{jobs_import_github_path}", import_path: "#{import_github_path}" } }
+= render 'import/githubish_status', provider: 'github'
diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml
index 0788924d44a..866b278ce57 100644
--- a/app/views/projects/new.html.haml
+++ b/app/views/projects/new.html.haml
@@ -69,6 +69,11 @@
                     = link_to new_import_fogbugz_path, class: 'btn import_fogbugz' do
                       = icon('bug', text: 'Fogbugz')
                 %div
+                  - if gitea_import_enabled?
+                    = link_to new_import_gitea_url, class: 'btn import_gitea' do
+                      = custom_icon('go_logo')
+                      Gitea
+                %div
                   - if git_import_enabled?
                     = link_to "#", class: 'btn js-toggle-button import_git' do
                       = icon('git', text: 'Repo by URL')
diff --git a/app/views/shared/icons/_go_logo.svg.erb b/app/views/shared/icons/_go_logo.svg.erb
new file mode 100644
index 00000000000..5052651c110
--- /dev/null
+++ b/app/views/shared/icons/_go_logo.svg.erb
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="<%= size %>" height="<%= size %>" viewBox="0 0 16 16"><g fill-rule="evenodd" transform="translate(0 1)"><path d="m14 15.01h1v-8.02c0-3.862-3.134-6.991-7-6.991-3.858 0-7 3.13-7 6.991v8.02h1v-8.02c0-3.306 2.691-5.991 6-5.991 3.314 0 6 2.682 6 5.991v8.02m-10.52-13.354c-.366-.402-.894-.655-1.48-.655-1.105 0-2 .895-2 2 0 .868.552 1.606 1.325 1.883.102-.321.226-.631.371-.93-.403-.129-.695-.507-.695-.953 0-.552.448-1 1-1 .306 0 .58.138.764.354.222-.25.461-.483.717-.699m9.04-.002c.366-.401.893-.653 1.479-.653 1.105 0 2 .895 2 2 0 .867-.552 1.606-1.324 1.883-.101-.321-.225-.632-.37-.931.403-.129.694-.507.694-.952 0-.552-.448-1-1-1-.305 0-.579.137-.762.353-.222-.25-.461-.483-.717-.699"/><path d="m5.726 7.04h1.557v.124c0 .283-.033.534-.1.752-.065.202-.175.391-.33.566-.35.394-.795.591-1.335.591-.527 0-.979-.19-1.355-.571-.376-.382-.564-.841-.564-1.377 0-.547.191-1.01.574-1.391.382-.382.848-.574 1.396-.574.295 0 .57.06.825.181.244.12.484.316.72.586l-.405.388c-.309-.412-.686-.618-1.13-.618-.399 0-.733.138-1 .413-.27.27-.405.609-.405 1.015 0 .42.151.766.452 1.037.282.252.587.378.915.378.28 0 .531-.094.754-.283.223-.19.347-.418.373-.683h-.94v-.535m2.884.061c0-.53.194-.986.583-1.367.387-.381.853-.571 1.396-.571.537 0 .998.192 1.382.576.386.384.578.845.578 1.384 0 .542-.194 1-.581 1.379-.389.379-.858.569-1.408.569-.487 0-.923-.168-1.311-.505-.426-.373-.64-.861-.64-1.465m.574.007c0 .417.14.759.42 1.028.278.269.6.403.964.403.395 0 .729-.137 1-.41.272-.277.408-.613.408-1.01 0-.402-.134-.739-.403-1.01-.267-.273-.597-.41-.991-.41-.392 0-.723.137-.993.41-.27.27-.405.604-.405 1m-.184 3.918c.525.026.812.063.812.063.271.025.324-.096.116-.273 0 0-.775-.813-1.933-.813-1.159 0-1.923.813-1.923.813-.211.174-.153.3.12.273 0 0 .286-.037.81-.063v.477c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.252.25c0 .268.224.5.5.5.268 0 .5-.223.5-.498v-.478m-1-1.023c.552 0 1-.224 1-.5 0-.276-.448-.5-1-.5-.552 0-1 .224-1 .5 0 .276.448.5 1 .5"/></g></svg>