diff options
21 files changed, 226 insertions, 64 deletions
diff --git a/CHANGELOG b/CHANGELOG index 52329ef928d..9636aa48f43 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -18,6 +18,7 @@ v 8.10.0 (unreleased) - Exclude email check from the standard health check - Fix changing issue state columns in milestone view - Add notification settings dropdown for groups + - Allow importing from Github using Personal Access Tokens. (Eric K Idema) - Fix user creation with stronger minimum password requirements !4054 (nathan-pmt) - PipelinesFinder uses git cache data - Check for conflicts with existing Project's wiki path when creating a new project. @@ -48,6 +49,7 @@ v 8.9.3 - Reduce overhead and optimize ProjectTeam#max_member_access performance. !4973 - Use update_columns to by_pass all the dirty code on active_record. !4985 - Decreased min width of screen to 1280px for pinned sidebar + - Fix restore Rake task warning message output !4980 - Fix encrypted data backwards compatibility after upgrading attr_encrypted gem - Update mobile button icons to be more inline with typical UI paradigms @@ -100,7 +102,6 @@ v 8.9.1 - Remove duplicate 'New Page' button on edit wiki page v 8.9.0 -v 8.9.0 (unreleased) - Fix group visibility form layout in application settings - Fix builds API response not including commit data - Fix error when CI job variables key specified but not defined diff --git a/app/assets/javascripts/application.js.coffee b/app/assets/javascripts/application.js.coffee index 05080d50ce7..20fe5a5cc27 100644 --- a/app/assets/javascripts/application.js.coffee +++ b/app/assets/javascripts/application.js.coffee @@ -269,7 +269,7 @@ $ -> new Aside() # Sidenav pinning - if $window.width() < 1280 and $.cookie('pin_nav') is 'true' + if $window.width() < 1024 and $.cookie('pin_nav') is 'true' $.cookie('pin_nav', 'false', { path: '/', expires: 365 * 10 }) $('.page-with-sidebar') .toggleClass('page-sidebar-collapsed page-sidebar-expanded') diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 87f8a17659f..211a9af2348 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -7,7 +7,7 @@ $gutter_collapsed_width: 62px; $gutter_width: 290px; $gutter_inner_width: 258px; $sidebar-transition-duration: .15s; -$sidebar-breakpoint: 1280px; +$sidebar-breakpoint: 1024px; /* * UI elements diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index 67bf4190e7e..9c1b0eb20f4 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -1,14 +1,29 @@ class Import::GithubController < Import::BaseController before_action :verify_github_import_enabled - before_action :github_auth, except: :callback + before_action :github_auth, only: [:status, :jobs, :create] rescue_from Octokit::Unauthorized, with: :github_unauthorized + helper_method :logged_in_with_github? + + def new + if logged_in_with_github? + go_to_github_for_permissions + elsif session[:github_access_token] + redirect_to status_import_github_url + end + end + def callback session[:github_access_token] = client.get_token(params[:code]) redirect_to status_import_github_url end + def personal_access_token + session[:github_access_token] = params[:personal_access_token] + redirect_to status_import_github_url + end + def status @repos = client.repos @already_added_projects = current_user.created_projects.where(import_type: "github") @@ -57,10 +72,14 @@ class Import::GithubController < Import::BaseController end def github_unauthorized - go_to_github_for_permissions + session[:github_access_token] = nil + redirect_to new_import_github_url, + alert: 'Access denied to your GitHub account.' end - private + def logged_in_with_github? + current_user.identities.exists?(provider: 'github') + end def access_params { github_access_token: session[:github_access_token] } diff --git a/app/views/import/github/new.html.haml b/app/views/import/github/new.html.haml new file mode 100644 index 00000000000..435ed7bd4cb --- /dev/null +++ b/app/views/import/github/new.html.haml @@ -0,0 +1,43 @@ +- page_title "GitHub Import" +- header_title "Projects", root_path + +%h3.page-title + = icon 'github', text: 'Import Projects from GitHub' + +- if github_import_configured? + %p + To import a GitHub project, you first need to authorize GitLab to access + the list of your GitHub repositories: + + = link_to 'List Your GitHub Repositories', status_import_github_path, class: 'btn btn-success' + + %hr + +%p + - if github_import_configured? + Alternatively, + - else + To import a GitHub project, + you can use a + = succeed '.' do + = link_to 'Personal Access Token', 'https://github.com/settings/tokens' + When you create your Personal Access Token, + you will need to select the <code>repo</code> scope, so we can display a + list of your public and private repositories which are available for import. + += form_tag personal_access_token_import_github_path, method: :post, class: 'form-inline' do + .form-group + = text_field_tag :personal_access_token, '', class: 'form-control', placeholder: "Personal Access Token", size: 40 + = submit_tag 'List Your GitHub Repositories', class: 'btn btn-success' + +- unless github_import_configured? + %hr + %p + Note: + - if current_user.admin? + As an administrator you may like to configure + - else + Consider asking your GitLab administrator to configure + = link_to 'GitHub integration', help_page_path("integration", "github") + which will allow login via GitHub and allow importing projects without + generating a Personal Access Token. diff --git a/app/views/projects/_github_import_modal.html.haml b/app/views/projects/_github_import_modal.html.haml deleted file mode 100644 index 46ad1559356..00000000000 --- a/app/views/projects/_github_import_modal.html.haml +++ /dev/null @@ -1,13 +0,0 @@ -%div#github_import_modal.modal - .modal-dialog - .modal-content - .modal-header - %a.close{href: "#", "data-dismiss" => "modal"} × - %h3 Import projects from GitHub - .modal-body - To enable importing projects from GitHub, - - if current_user.admin? - as administrator you need to configure - - else - ask your Gitlab administrator to configure - == #{link_to 'OAuth integration', help_page_path("integration", "github")}. diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 8a73b077357..05f33b78a47 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -23,6 +23,7 @@ .input-group-addon = root_url = f.select :namespace_id, namespaces_options(params[:namespace_id] || :current_user, display_path: true), {}, {class: 'select2 js-select-namespace', tabindex: 1} + - else .input-group-addon.static-namespace #{root_url}#{current_user.username}/ @@ -44,15 +45,8 @@ .col-sm-12.import-buttons %div - if github_import_enabled? - - if github_import_configured? - = link_to status_import_github_path, class: 'btn import_github' do - %i.fa.fa-github - GitHub - - else - = link_to '#', class: 'how_to_import_link btn import_github' do - %i.fa.fa-github - GitHub - = render 'github_import_modal' + = link_to new_import_github_path, class: 'btn import_github' do + = icon 'github', text: 'GitHub' %div - if bitbucket_import_enabled? - if bitbucket_import_configured? diff --git a/config/initializers/rack_attack.rb.example b/config/initializers/rack_attack.rb.example index 30d05f16153..69052c029f2 100644 --- a/config/initializers/rack_attack.rb.example +++ b/config/initializers/rack_attack.rb.example @@ -10,7 +10,8 @@ paths_to_be_protected = [ "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session", "#{Rails.application.config.relative_url_root}/users", "#{Rails.application.config.relative_url_root}/users/confirmation", - "#{Rails.application.config.relative_url_root}/unsubscribes/" + "#{Rails.application.config.relative_url_root}/unsubscribes/", + "#{Rails.application.config.relative_url_root}/import/github/personal_access_token" ] diff --git a/config/routes.rb b/config/routes.rb index 2aab73720f2..c04780fec88 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -139,6 +139,7 @@ Rails.application.routes.draw do # namespace :import do resource :github, only: [:create, :new], controller: :github do + post :personal_access_token get :status get :callback get :jobs diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md index a7dfac2c120..a2b2a4b88f9 100644 --- a/doc/workflow/importing/import_projects_from_github.md +++ b/doc/workflow/importing/import_projects_from_github.md @@ -1,8 +1,10 @@ # Import your project from GitHub to GitLab
>**Note:**
-In order to enable the GitHub import setting, you should first
-enable the [GitHub integration][gh-import] in your GitLab instance.
+In order to enable the GitHub import setting, you may also want to
+enable the [GitHub integration][gh-import] in your GitLab instance. This
+configuration is optional, you will be able import your GitHub repositories
+with a Personal Access Token.
At its current state, GitHub importer can import:
@@ -20,9 +22,15 @@ It is not yet possible to import your cross-repository pull requests (those from forks). We are working on improving this in the near future.
The importer page is visible when you [create a new project][new-project].
-Click on the **GitHub** link and you will be redirected to GitHub for
-permission to access your projects. After accepting, you'll be automatically
-redirected to the importer.
+Click on the **GitHub** link and, if you are logged in via the GitHub
+integration, you will be redirected to GitHub for permission to access your
+projects. After accepting, you'll be automatically redirected to the importer.
+
+If you are not using the GitHub integration, you can still perform a one-off
+authorization with GitHub to access your projects.
+
+Alternatively, you can also enter a GitHub Personal Access Token. Once you enter
+your token, you'll be taken to the importer.
diff --git a/features/dashboard/new_project.feature b/features/dashboard/new_project.feature index 56b4a639c01..8ddafb6a7ac 100644 --- a/features/dashboard/new_project.feature +++ b/features/dashboard/new_project.feature @@ -21,7 +21,7 @@ Background: Scenario: I should see instructions on how to import from GitHub Given I see "New Project" page When I click on "Import project from GitHub" - Then I see instructions on how to import from GitHub + Then I am redirected to the GitHub import page @javascript Scenario: I should see Google Code import page diff --git a/features/steps/dashboard/new_project.rb b/features/steps/dashboard/new_project.rb index 31f8924c38c..09373168dad 100644 --- a/features/steps/dashboard/new_project.rb +++ b/features/steps/dashboard/new_project.rb @@ -28,14 +28,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps first('.import_github').click end - step 'I see instructions on how to import from GitHub' do - github_modal = first('.modal-body') - expect(github_modal).to be_visible - expect(github_modal).to have_content "To enable importing projects from GitHub" - - page.all('.modal-body').each do |element| - expect(element).not_to be_visible unless element == github_modal - end + step 'I am redirected to the GitHub import page' do + expect(current_path).to eq new_import_github_path end step 'I click on "Repo by URL"' do diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 81d66271136..d77a5e3ff09 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -160,11 +160,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attribute( - original: link_text || match, - project: project.id, - object_sym => object.id - ) + data = data_attributes_for(link_text || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -183,6 +179,14 @@ module Banzai end end + def data_attributes_for(text, project, object) + data_attribute( + original: text, + project: project.id, + object_sym => object.id + ) + end + def object_link_text_extras(object, matches) extras = [] diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 5351272f42d..4042e9a4c25 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -46,6 +46,26 @@ module Banzai end end + def object_link_title(object) + if object.is_a?(ExternalIssue) + "Issue in #{object.project.external_issue_tracker.title}" + else + super + end + end + + def data_attributes_for(text, project, object) + if object.is_a?(ExternalIssue) + data_attribute( + project: project.id, + external_issue: object.id, + reference_type: ExternalIssueReferenceFilter.reference_type + ) + else + super + end + end + def find_projects_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d6f34c9cd8..bf058241cda 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -29,7 +29,7 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] = self.class.reference_type + attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..043f10d96a9 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,7 +69,11 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9ee72fde92f..b43ee5b3383 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,12 +33,13 @@ namespace :gitlab do unless backup.skipped?('db') unless ENV['force'] == 'yes' - warning = warning = <<-MSG.strip_heredoc + warning = <<-MSG.strip_heredoc Before restoring the database we recommend removing all existing tables to avoid future upgrade problems. Be aware that if you have custom tables in the GitLab database these tables and all data will be removed. MSG + puts warning.color(:red) ask_to_continue puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) diff --git a/spec/controllers/import/github_controller_spec.rb b/spec/controllers/import/github_controller_spec.rb index c55a3c28208..51d59526854 100644 --- a/spec/controllers/import/github_controller_spec.rb +++ b/spec/controllers/import/github_controller_spec.rb @@ -16,6 +16,24 @@ describe Import::GithubController do allow(controller).to receive(:github_import_enabled?).and_return(true) end + describe "GET new" do + it "redirects to GitHub for an access token if logged in with GitHub" do + allow(controller).to receive(:logged_in_with_github?).and_return(true) + expect(controller).to receive(:go_to_github_for_permissions) + + get :new + end + + it "redirects to status if we already have a token" do + assign_session_token + allow(controller).to receive(:logged_in_with_github?).and_return(false) + + get :new + + expect(controller).to redirect_to(status_import_github_url) + end + end + describe "GET callback" do it "updates access token" do token = "asdasd12345" @@ -32,6 +50,20 @@ describe Import::GithubController do end end + describe "POST personal_access_token" do + it "updates access token" do + token = "asdfasdf9876" + + allow_any_instance_of(Gitlab::GithubImport::Client). + to receive(:user).and_return(true) + + post :personal_access_token, personal_access_token: token + + expect(session[:github_access_token]).to eq(token) + expect(controller).to redirect_to(status_import_github_url) + end + end + describe "GET status" do before do @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim') @@ -59,6 +91,17 @@ describe Import::GithubController do expect(assigns(:already_added_projects)).to eq([@project]) expect(assigns(:repos)).to eq([]) end + + it "handles an invalid access token" do + allow_any_instance_of(Gitlab::GithubImport::Client). + to receive(:repos).and_raise(Octokit::Unauthorized) + + get :status + + expect(session[:github_access_token]).to eq(nil) + expect(controller).to redirect_to(new_import_github_url) + expect(flash[:alert]).to eq('Access denied to your GitHub account.') + end end describe "POST create" do diff --git a/spec/lib/banzai/filter/issue_reference_filter_spec.rb b/spec/lib/banzai/filter/issue_reference_filter_spec.rb index 8d6ce114aa9..a005b4990e7 100644 --- a/spec/lib/banzai/filter/issue_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/issue_reference_filter_spec.rb @@ -199,6 +199,19 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do end end + context 'referencing external issues' do + let(:project) { create(:redmine_project) } + + it 'renders internal issue IDs as external issue links' do + doc = reference_filter('#1') + link = doc.css('a').first + + expect(link.attr('data-reference-type')).to eq('external_issue') + expect(link.attr('title')).to eq('Issue in Redmine') + expect(link.attr('data-external-issue')).to eq('1') + end + end + describe '#issues_per_Project' do context 'using an internal issue tracker' do it 'returns a Hash containing the issues per project' do diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 7c21cbe96d9..3b023a35446 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -20,6 +20,20 @@ describe Gitlab::GithubImport::Client, lib: true do expect { client.api }.not_to raise_error end + context 'when config is missing' do + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([]) + end + + it 'is still possible to get an Octokit client' do + expect { client.api }.not_to raise_error + end + + it 'is not be possible to get an OAuth2 client' do + expect { client.client }.to raise_error(Projects::ImportService::Error) + end + end + context 'allow SSL verification to be configurable on API' do before do github_provider['verify_ssl'] = false diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 1ceb7c3e6ff..48d374883d7 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -350,6 +350,8 @@ describe GitPushService, services: true do it "doesn't close issues when external issue tracker is in use" do allow_any_instance_of(Project).to receive(:default_issues_tracker?). and_return(false) + external_issue_tracker = double(title: 'My Tracker', issue_path: issue.iid) + allow_any_instance_of(Project).to receive(:external_issue_tracker).and_return(external_issue_tracker) # The push still shouldn't create cross-reference notes. expect do |