diff options
author | Valery Sizov <valery@gitlab.com> | 2015-01-27 15:37:19 -0800 |
---|---|---|
committer | Valery Sizov <valery@gitlab.com> | 2015-02-05 12:50:34 -0800 |
commit | 5194214e3a2f97accf0c8119b4cb39fd4fcef5db (patch) | |
tree | 7320257c4470abadd90a0f17fd92f408143b6bb1 | |
parent | 0a9cab4ee65f2b42c56989698c401cab60d68b53 (diff) | |
download | gitlab-ce-5194214e3a2f97accf0c8119b4cb39fd4fcef5db.tar.gz |
GitLab integration. Importer
23 files changed, 514 insertions, 16 deletions
@@ -29,6 +29,7 @@ gem 'omniauth-twitter' gem 'omniauth-github' gem 'omniauth-shibboleth' gem 'omniauth-kerberos' +gem 'omniauth-gitlab' gem 'doorkeeper', '2.1.0' gem "rack-oauth2", "~> 1.0.5" diff --git a/Gemfile.lock b/Gemfile.lock index 7f115d79de0..6d2d281e476 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -332,6 +332,9 @@ GEM omniauth-github (1.1.1) omniauth (~> 1.0) omniauth-oauth2 (~> 1.1) + omniauth-gitlab (1.0.0) + omniauth (~> 1.0) + omniauth-oauth2 (~> 1.0) omniauth-google-oauth2 (0.2.5) omniauth (> 1.0) omniauth-oauth2 (~> 1.1) @@ -689,6 +692,7 @@ DEPENDENCIES octokit (= 3.7.0) omniauth (~> 1.1.3) omniauth-github + omniauth-gitlab omniauth-google-oauth2 omniauth-kerberos omniauth-shibboleth diff --git a/app/assets/images/authbuttons/gitlab_32.png b/app/assets/images/authbuttons/gitlab_32.png Binary files differnew file mode 100644 index 00000000000..f3b78cb6efb --- /dev/null +++ b/app/assets/images/authbuttons/gitlab_32.png diff --git a/app/assets/images/authbuttons/gitlab_64.png b/app/assets/images/authbuttons/gitlab_64.png Binary files differnew file mode 100644 index 00000000000..ff2945fe89e --- /dev/null +++ b/app/assets/images/authbuttons/gitlab_64.png diff --git a/app/controllers/github_imports_controller.rb b/app/controllers/importers/githubs_controller.rb index b73e3f7ffac..5bb64c4a6ce 100644 --- a/app/controllers/github_imports_controller.rb +++ b/app/controllers/importers/githubs_controller.rb @@ -1,4 +1,4 @@ -class GithubImportsController < ApplicationController +class Importers::GithubsController < ApplicationController before_filter :github_auth, except: :callback rescue_from Octokit::Unauthorized, with: :github_unauthorized @@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController token = client.auth_code.get_token(params[:code]).token current_user.github_access_token = token current_user.save - redirect_to status_github_import_url + redirect_to status_importers_github_url end def status @@ -69,7 +69,7 @@ class GithubImportsController < ApplicationController def go_to_github_for_permissions redirect_to client.auth_code.authorize_url({ - redirect_uri: callback_github_import_url, + redirect_uri: callback_importers_github_url, scope: "repo, user, user:email" }) end diff --git a/app/controllers/importers/gitlabs_controller.rb b/app/controllers/importers/gitlabs_controller.rb new file mode 100644 index 00000000000..d020c870a4f --- /dev/null +++ b/app/controllers/importers/gitlabs_controller.rb @@ -0,0 +1,69 @@ +class Importers::GitlabsController < ApplicationController + before_filter :gitlab_auth, except: :callback + + rescue_from OAuth2::Error, with: :gitlab_unauthorized + + def callback + token = client.get_token(params[:code], callback_importers_gitlab_url) + current_user.gitlab_access_token = token + current_user.save + redirect_to status_importers_gitlab_url + end + + def status + @repos = client.projects + + @already_added_projects = current_user.created_projects.where(import_type: "gitlab") + already_added_projects_names = @already_added_projects.pluck(:import_source) + + @repos.to_a.reject!{|repo| already_added_projects_names.include? repo["path_with_namespace"]} + end + + def jobs + jobs = current_user.created_projects.where(import_type: "gitlab").to_json(:only => [:id, :import_status]) + render json: jobs + end + + def create + @repo_id = params[:repo_id].to_i + repo = client.project(@repo_id) + target_namespace = params[:new_namespace].presence || repo["namespace"]["path"] + existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace) + + if existing_namespace + if existing_namespace.owner == current_user + namespace = existing_namespace + else + @already_been_taken = true + @target_namespace = target_namespace + @project_name = repo["path"] + render and return + end + else + namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user) + namespace.add_owner(current_user) + end + + @project = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, current_user).execute + end + + private + + def client + @client ||= Gitlab::GitlabImport::Client.new(current_user.gitlab_access_token) + end + + def gitlab_auth + if current_user.gitlab_access_token.blank? + go_to_gitlab_for_permissions + end + end + + def go_to_gitlab_for_permissions + redirect_to client.authorize_url(callback_importers_gitlab_url) + end + + def gitlab_unauthorized + go_to_gitlab_for_permissions + end +end diff --git a/app/helpers/oauth_helper.rb b/app/helpers/oauth_helper.rb index df18db71c84..c7bc9307a58 100644 --- a/app/helpers/oauth_helper.rb +++ b/app/helpers/oauth_helper.rb @@ -4,7 +4,7 @@ module OauthHelper end def default_providers - [:twitter, :github, :google_oauth2, :ldap] + [:twitter, :github, :gitlab, :google_oauth2, :ldap] end def enabled_oauth_providers @@ -13,7 +13,7 @@ module OauthHelper def enabled_social_providers enabled_oauth_providers.select do |name| - [:twitter, :github, :google_oauth2].include?(name.to_sym) + [:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym) end end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 5cec6ae99d8..36463892ebf 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -253,4 +253,8 @@ module ProjectsHelper def github_import_enabled? enabled_oauth_providers.include?(:github) end + + def gitlab_import_enabled? + enabled_oauth_providers.include?(:gitlab) + end end diff --git a/app/views/github_imports/create.js.haml b/app/views/importers/githubs/create.js.haml index cd4c9fbf360..cd4c9fbf360 100644 --- a/app/views/github_imports/create.js.haml +++ b/app/views/importers/githubs/create.js.haml diff --git a/app/views/github_imports/status.html.haml b/app/views/importers/githubs/status.html.haml index 52a1e16cd04..1c7e8209e6e 100644 --- a/app/views/github_imports/status.html.haml +++ b/app/views/importers/githubs/status.html.haml @@ -43,11 +43,11 @@ if tr.find(".import-target input").length > 0 new_namespace = tr.find(".import-target input").prop("value") tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name")) - $.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script' + $.post "#{importers_github_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script' setInterval (-> - $.get "#{jobs_github_import_path}", (data)-> + $.get "#{jobs_importers_github_path}", (data)-> $.each data, (i, job) -> job_item = $("#project_" + job.id) status_field = job_item.find(".job-status") diff --git a/app/views/importers/gitlabs/create.js.haml b/app/views/importers/gitlabs/create.js.haml new file mode 100644 index 00000000000..cd4c9fbf360 --- /dev/null +++ b/app/views/importers/gitlabs/create.js.haml @@ -0,0 +1,18 @@ +- if @already_been_taken + :plain + target_field = $("tr#repo_#{@repo_id} .import-target") + origin_target = target_field.text() + project_name = "#{@project_name}" + origin_namespace = "#{@target_namespace}" + target_field.empty() + target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>") + target_field.append("<input type='text' name='target_namespace' />") + target_field.append("/" + project_name) + target_field.data("project_name", project_name) + target_field.find('input').prop("value", origin_namespace) +- else + :plain + job = $("tr#repo_#{@repo_id}") + job.attr("id", "project_#{@project.id}") + $("table.import-jobs tbody").prepend(job) + job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started") diff --git a/app/views/importers/gitlabs/status.html.haml b/app/views/importers/gitlabs/status.html.haml new file mode 100644 index 00000000000..493c938cadb --- /dev/null +++ b/app/views/importers/gitlabs/status.html.haml @@ -0,0 +1,63 @@ +%h3.page-title + %i.fa.fa-github + Import repositories from GitLab.com + +%p.light + Select projects you want to import. + +%hr +%table.table.import-jobs + %thead + %tr + %th From GitLab.com + %th To GitLab private instance + %th Status + %tbody + - @already_added_projects.each do |project| + %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"} + %td= project.import_source + %td + %strong= link_to project.name_with_namespace, project + %td.job-status + - if project.import_status == 'finished' + %span.cgreen + %i.fa.fa-check + done + - else + = project.human_import_status_name + + - @repos.each do |repo| + %tr{id: "repo_#{repo["id"]}"} + %td= repo["path_with_namespace"] + %td.import-target + = repo["path_with_namespace"] + %td.import-actions.job-status + = button_tag "Add", class: "btn btn-add-to-import" + + +:coffeescript + $(".btn-add-to-import").click () -> + new_namespace = null + tr = $(this).closest("tr") + id = tr.attr("id").replace("repo_", "") + if tr.find(".import-target input").length > 0 + new_namespace = tr.find(".import-target input").prop("value") + tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name")) + $.post "#{importers_gitlab_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script' + + + setInterval (-> + $.get "#{jobs_importers_gitlab_path}", (data)-> + $.each data, (i, job) -> + job_item = $("#project_" + job.id) + status_field = job_item.find(".job-status") + + if job.import_status == 'finished' + job_item.removeClass("active").addClass("success") + status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>') + else if job.import_status == 'started' + status_field.html("<i class='fa fa-spinner fa-spin'></i> started") + else + status_field.html(job.import_status) + + ), 4000 diff --git a/app/views/projects/_gitlab_import_modal.html.haml b/app/views/projects/_gitlab_import_modal.html.haml new file mode 100644 index 00000000000..d402098cbd4 --- /dev/null +++ b/app/views/projects/_gitlab_import_modal.html.haml @@ -0,0 +1,22 @@ +%div#gitlab_import_modal.modal.hide + .modal-dialog + .modal-content + .modal-header + %a.close{href: "#", "data-dismiss" => "modal"} × + %h3 GitLab OAuth import + .modal-body + You need to setup integration with GitLab first. + = link_to 'How to setup integration with GitLab', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/gitlab.md' + + +:javascript + $(function(){ + var import_modal = $('#gitlab_import_modal').modal({modal: true, show:false}); + $('.how_to_import_link').bind("click", function(e){ + e.preventDefault(); + import_modal.show(); + }); + $('.modal-header .close').bind("click", function(){ + import_modal.hide(); + }) + }) diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 3e0f9cbd80b..ae1dd88b696 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -44,7 +44,7 @@ .col-sm-2 .col-sm-10 - if github_import_enabled? - = link_to status_github_import_path do + = link_to status_importers_github_path do %i.fa.fa-github Import projects from GitHub - else @@ -52,6 +52,19 @@ %i.fa.fa-github Import projects from GitHub = render 'github_import_modal' + + .project-import.form-group + .col-sm-2 + .col-sm-10 + - if gitlab_import_enabled? + = link_to status_importers_gitlab_path do + %i.fa.fa-heart + Import projects from GitLab.com + - else + = link_to '#', class: 'how_to_import_link light' do + %i.fa.fa-heart + Import projects from GitLab.com + = render 'gitlab_import_modal' %hr.prepend-botton-10 diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb index 0bcc42bc62c..1ceea7ff07f 100644 --- a/app/workers/repository_import_worker.rb +++ b/app/workers/repository_import_worker.rb @@ -12,6 +12,8 @@ class RepositoryImportWorker if project.import_type == 'github' result_of_data_import = Gitlab::Github::Importer.new(project).execute + elsif project.import_type == 'gitlab' + result_of_data_import = Gitlab::GitlabImport::Importer.new(project).execute else result_of_data_import = true end diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index e9b843e29b4..9da7ebf4290 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -27,7 +27,7 @@ Doorkeeper.configure do # Access token expiration time (default 2 hours). # If you want to disable expiration, set this to nil. - # access_token_expires_in 2.hours + access_token_expires_in nil # Reuse access token for the same resource owner within an application (disabled by default) # Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383 diff --git a/config/routes.rb b/config/routes.rb index f0abd876ecd..fde76b16064 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -51,14 +51,25 @@ Gitlab::Application.routes.draw do end get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ } + # - # Github importer area + # Importers # - resource :github_import, only: [:create, :new] do - get :status - get :callback - get :jobs + namespace :importers do + resource :github, only: [:create, :new] do + get :status + get :callback + get :jobs + end + + resource :gitlab, only: [:create, :new] do + get :status + get :callback + get :jobs + end end + + # # Explore area diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md new file mode 100644 index 00000000000..47f187b021c --- /dev/null +++ b/doc/integration/gitlab.md @@ -0,0 +1,54 @@ +# GitLab OAuth2 OmniAuth Provider + +To enable the GitLab OmniAuth provider you must register your application with GitLab. GitLab will generate a client ID and secret key for you to use. + +1. Sign in to GitLab. + +1. Navigate to your settings. + +1. Select "Applications" in the left menu. + +1. Select "New application". + +1. Provide the required details. + - Name: This can be anything. Consider something like "\<Organization\>'s GitLab" or "\<Your Name\>'s GitLab" or something else descriptive. + - Redirect URI: + + ``` + http://gitlab.example.com/importers/gitlab/callback + http://gitlab.example.com/users/auth/gitlab/callback + ``` + + The first link is required for the importer and second for the authorization. + +1. Select "Submit". + +1. You should now see a Application ID and Secret. Keep this page open as you continue configuration. + +1. On your GitLab server, open the configuration file. + + ```sh + cd /home/git/gitlab + + sudo -u git -H editor config/gitlab.yml + ``` + +1. Find the section dealing with OmniAuth. See [Initial OmniAuth Configuration](README.md#initial-omniauth-configuration) for more details. + +1. Under `providers:` uncomment (or add) lines that look like the following: + + ``` + - { name: 'gitlab', app_id: 'YOUR APP ID', + app_secret: 'YOUR APP SECRET', + args: { scope: 'api' } } + ``` + +1. Change 'YOUR APP ID' to the Application ID from the GitLab application page. + +1. Change 'YOUR APP SECRET' to the secret from the GitLab application page. + +1. Save the configuration file. + +1. Restart GitLab for the changes to take effect. + +On the sign in page there should now be a GitLab icon below the regular sign in form. Click the icon to begin the authentication process. GitLab will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to your GitLab instance and will be signed in. diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb new file mode 100644 index 00000000000..64e369e9c12 --- /dev/null +++ b/lib/gitlab/gitlab_import/client.rb @@ -0,0 +1,82 @@ +module Gitlab + module GitlabImport + class Client + attr_reader :client, :api + + PER_PAGE = 100 + + def initialize(access_token) + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options + ) + + if access_token + @api = OAuth2::AccessToken.from_hash(@client, :access_token => access_token) + end + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "api" + }) + end + + def get_token(code, redirect_uri) + client.auth_code.get_token(code, redirect_uri: redirect_uri).token + end + + def issues(project_identifier) + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + def issue_comments(project_identifier, issue_id) + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + def project(id) + api.get("/api/v3/projects/#{id}").parsed + end + + def projects + lazy_page_iterator(PER_PAGE) do |page| + api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + end + end + + private + + def lazy_page_iterator(per_page) + Enumerator.new do |y| + page = 1 + loop do + items = yield(page) + items.each do |item| + y << item + end + break if items.empty? || items.size < per_page + page += 1 + end + end + end + + def config + Gitlab.config.omniauth.providers.select{|provider| provider.name == "gitlab"}.first + end + + def github_options + { + site: 'https://gitlab.com/', + authorize_url: 'oauth/authorize', + token_url: 'oauth/token' + } + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb new file mode 100644 index 00000000000..3e9087a556c --- /dev/null +++ b/lib/gitlab/gitlab_import/importer.rb @@ -0,0 +1,48 @@ +module Gitlab + module GitlabImport + class Importer + attr_reader :project, :client + + def initialize(project) + @project = project + @client = Client.new(project.creator.gitlab_access_token) + end + + def execute + project_identifier = URI.encode(project.import_source, '/') + + #Issues && Comments + issues = client.issues(project_identifier) + + issues.each do |issue| + body = "*Created by: #{issue["author"]["name"]}*\n\n#{issue["description"]}" + + + comments = client.issue_comments(project_identifier, issue["id"]) + if comments.any? + body += "\n\n\n**Imported comments:**\n" + end + comments.each do |comment| + body += "\n\n*By #{comment["author"]["name"]} on #{comment["created_at"]}*\n\n#{comment["body"]}" + end + + project.issues.create!( + description: body, + title: issue["title"], + state: issue["state"], + author_id: gl_user_id(project, issue["author"]["id"]) + ) + end + + true + end + + private + + def gl_user_id(project, gitlab_id) + user = User.joins(:identities).find_by("identities.extern_uid = ?", gitlab_id.to_s) + (user && user.id) || project.creator_id + end + end + end +end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb new file mode 100644 index 00000000000..affd828e816 --- /dev/null +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitlabImport + class ProjectCreator + attr_reader :repo, :namespace, :current_user + + def initialize(repo, namespace, current_user) + @repo = repo + @namespace = namespace + @current_user = current_user + end + + def execute + @project = Project.new( + name: repo["name"], + path: repo["path"], + description: repo["description"], + namespace: namespace, + creator: current_user, + visibility_level: repo["visibility_level"], + import_type: "gitlab", + import_source: repo["path_with_namespace"], + import_url: repo["http_url_to_repo"]#.sub("://", "://oauth2@#{current_user.gitlab_access_token}") + ) + + if @project.save! + @project.reload + + if @project.import_failed? + @project.import_retry + else + @project.import_start + end + end + + @project + end + end + end +end diff --git a/spec/controllers/github_imports_controller_spec.rb b/spec/controllers/importers/githubs_controller_spec.rb index 26e7854fea3..e21b5f8a470 100644 --- a/spec/controllers/github_imports_controller_spec.rb +++ b/spec/controllers/importers/githubs_controller_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe GithubImportsController do +describe Importers::GithubsController do let(:user) { create(:user, github_access_token: 'asd123') } before do @@ -16,7 +16,7 @@ describe GithubImportsController do get :callback user.reload.github_access_token.should == token - controller.should redirect_to(status_github_import_url) + controller.should redirect_to(status_importers_github_url) end end diff --git a/spec/controllers/importers/gitlabs_controller_spec.rb b/spec/controllers/importers/gitlabs_controller_spec.rb new file mode 100644 index 00000000000..af42d14ded3 --- /dev/null +++ b/spec/controllers/importers/gitlabs_controller_spec.rb @@ -0,0 +1,68 @@ +require 'spec_helper' + +describe Importers::GitlabsController do + let(:user) { create(:user, gitlab_access_token: 'asd123') } + + before do + sign_in(user) + end + + describe "GET callback" do + it "updates access token" do + token = "asdasd12345" + Gitlab::GitlabImport::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token) + Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "gitlab") + + get :callback + + user.reload.gitlab_access_token.should == token + controller.should redirect_to(status_importers_gitlab_url) + end + end + + describe "GET status" do + before do + @repo = OpenStruct.new(path: 'vim', path_with_namespace: 'asd/vim') + end + + it "assigns variables" do + @project = create(:project, import_type: 'gitlab', creator_id: user.id) + controller.stub_chain(:client, :projects).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([@repo]) + end + + it "does not show already added project" do + @project = create(:project, import_type: 'gitlab', creator_id: user.id, import_source: 'asd/vim') + controller.stub_chain(:client, :projects).and_return([@repo]) + + get :status + + expect(assigns(:already_added_projects)).to eq([@project]) + expect(assigns(:repos)).to eq([]) + end + end + + describe "POST create" do + before do + @repo = { + path: 'vim', + path_with_namespace: 'asd/vim', + owner: {name: "john"}, + namespace: {path: "john"} + }.with_indifferent_access + end + + it "takes already existing namespace" do + namespace = create(:namespace, name: "john", owner: user) + Gitlab::GitlabImport::ProjectCreator.should_receive(:new).with(@repo, namespace, user). + and_return(double(execute: true)) + controller.stub_chain(:client, :project).and_return(@repo) + + post :create, format: :js + end + end +end |