diff options
author | Douglas Barbosa Alexandre <dbalexandre@gmail.com> | 2016-12-16 23:25:14 +0000 |
---|---|---|
committer | Douglas Barbosa Alexandre <dbalexandre@gmail.com> | 2016-12-16 23:25:14 +0000 |
commit | a22be3065dc9c80d3a18ca81eda480b9c39e35a6 (patch) | |
tree | d29fc79464dbf3f1fd09d034a4b73c8eb230a9e3 | |
parent | 12a7e717d7b9fdd265d54a9c5bd07394e304b187 (diff) | |
parent | a3be4aeb7a71cc940394a5f13d09e79fcafdb1d5 (diff) | |
download | gitlab-ce-a22be3065dc9c80d3a18ca81eda480b9c39e35a6.tar.gz |
Merge branch 'bitbucket-oauth2' into 'master'
Refactor Bitbucket importer to use BitBucket API Version 2
## What does this MR do?
## Are there points in the code the reviewer needs to double check?
## Why was this MR needed?
## What are the relevant issue numbers?
https://gitlab.com/gitlab-org/gitlab-ce/issues/19946
## Screenshots (if relevant)
This MR needs the following permissions in the Bitbucket OAuth settings:
![image](/uploads/a26ae5e430a724bf581a92da7028ce3c/image.png)
- []
## Does this MR meet the acceptance criteria?
- [ ] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [ ] API support added
- Tests
- [ ] Added for this feature/bug
- [ ] All builds are passing
- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [ ] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)
See merge request !5995
56 files changed, 1271 insertions, 539 deletions
@@ -22,7 +22,6 @@ gem 'doorkeeper', '~> 4.2.0' gem 'omniauth', '~> 1.3.1' gem 'omniauth-auth0', '~> 1.4.1' gem 'omniauth-azure-oauth2', '~> 0.0.6' -gem 'omniauth-bitbucket', '~> 0.0.2' gem 'omniauth-cas3', '~> 1.1.2' gem 'omniauth-facebook', '~> 4.0.0' gem 'omniauth-github', '~> 1.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index 23e45ddc16f..811adfc5c1d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -432,10 +432,6 @@ GEM jwt (~> 1.0) omniauth (~> 1.0) omniauth-oauth2 (~> 1.1) - omniauth-bitbucket (0.0.2) - multi_json (~> 1.7) - omniauth (~> 1.1) - omniauth-oauth (~> 1.0) omniauth-cas3 (1.1.3) addressable (~> 2.3) nokogiri (~> 1.6.6) @@ -902,7 +898,6 @@ DEPENDENCIES omniauth (~> 1.3.1) omniauth-auth0 (~> 1.4.1) omniauth-azure-oauth2 (~> 0.0.6) - omniauth-bitbucket (~> 0.0.2) omniauth-cas3 (~> 1.1.2) omniauth-facebook (~> 4.0.0) omniauth-github (~> 1.1.1) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index bcc0b17bce2..4df80195ae1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -262,7 +262,7 @@ class ApplicationController < ActionController::Base end def bitbucket_import_configured? - Gitlab::OAuth::Provider.enabled?(:bitbucket) && Gitlab::BitbucketImport.public_key.present? + Gitlab::OAuth::Provider.enabled?(:bitbucket) end def google_code_import_enabled? diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb index 6ea54744da8..8e42cdf415f 100644 --- a/app/controllers/import/bitbucket_controller.rb +++ b/app/controllers/import/bitbucket_controller.rb @@ -2,50 +2,57 @@ class Import::BitbucketController < Import::BaseController before_action :verify_bitbucket_import_enabled before_action :bitbucket_auth, except: :callback - rescue_from OAuth::Error, with: :bitbucket_unauthorized - rescue_from Gitlab::BitbucketImport::Client::Unauthorized, with: :bitbucket_unauthorized + rescue_from OAuth2::Error, with: :bitbucket_unauthorized + rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized def callback - request_token = session.delete(:oauth_request_token) - raise "Session expired!" if request_token.nil? + response = client.auth_code.get_token(params[:code], redirect_uri: callback_import_bitbucket_url) - request_token.symbolize_keys! - - access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url) - - session[:bitbucket_access_token] = access_token.token - session[:bitbucket_access_token_secret] = access_token.secret + session[:bitbucket_token] = response.token + session[:bitbucket_expires_at] = response.expires_at + session[:bitbucket_expires_in] = response.expires_in + session[:bitbucket_refresh_token] = response.refresh_token redirect_to status_import_bitbucket_url end def status - @repos = client.projects - @incompatible_repos = client.incompatible_projects + bitbucket_client = Bitbucket::Client.new(credentials) + repos = bitbucket_client.repos + + @repos, @incompatible_repos = repos.partition { |repo| repo.valid? } - @already_added_projects = current_user.created_projects.where(import_type: "bitbucket") + @already_added_projects = current_user.created_projects.where(import_type: 'bitbucket') already_added_projects_names = @already_added_projects.pluck(:import_source) - @repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" } + @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) } end def jobs - jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status]) - render json: jobs + render json: current_user.created_projects + .where(import_type: 'bitbucket') + .to_json(only: [:id, :import_status]) end def create + bitbucket_client = Bitbucket::Client.new(credentials) + @repo_id = params[:repo_id].to_s - repo = client.project(@repo_id.gsub('___', '/')) - @project_name = repo['slug'] - @target_namespace = find_or_create_namespace(repo['owner'], client.user['user']['username']) + name = @repo_id.gsub('___', '/') + repo = bitbucket_client.repo(name) + @project_name = params[:new_name].presence || repo.name - unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user, access_params).execute - render 'deploy_key' and return - end + repo_owner = repo.owner + repo_owner = current_user.username if repo_owner == bitbucket_client.user.username + @target_namespace = params[:new_namespace].presence || repo_owner + + namespace = find_or_create_namespace(@target_namespace, current_user) - if current_user.can?(:create_projects, @target_namespace) - @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @target_namespace, current_user, access_params).execute + if current_user.can?(:create_projects, namespace) + # The token in a session can be expired, we need to get most recent one because + # Bitbucket::Connection class refreshes it. + session[:bitbucket_token] = bitbucket_client.connection.token + @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @project_name, namespace, current_user, credentials).execute else render 'unauthorized' end @@ -54,8 +61,15 @@ class Import::BitbucketController < Import::BaseController private def client - @client ||= Gitlab::BitbucketImport::Client.new(session[:bitbucket_access_token], - session[:bitbucket_access_token_secret]) + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys end def verify_bitbucket_import_enabled @@ -63,26 +77,23 @@ class Import::BitbucketController < Import::BaseController end def bitbucket_auth - if session[:bitbucket_access_token].blank? - go_to_bitbucket_for_permissions - end + go_to_bitbucket_for_permissions if session[:bitbucket_token].blank? end def go_to_bitbucket_for_permissions - request_token = client.request_token(callback_import_bitbucket_url) - session[:oauth_request_token] = request_token - - redirect_to client.authorize_url(request_token, callback_import_bitbucket_url) + redirect_to client.auth_code.authorize_url(redirect_uri: callback_import_bitbucket_url) end def bitbucket_unauthorized go_to_bitbucket_for_permissions end - def access_params + def credentials { - bitbucket_access_token: session[:bitbucket_access_token], - bitbucket_access_token_secret: session[:bitbucket_access_token_secret] + token: session[:bitbucket_token], + expires_at: session[:bitbucket_expires_at], + expires_in: session[:bitbucket_expires_in], + refresh_token: session[:bitbucket_refresh_token] } end end diff --git a/app/views/import/bitbucket/status.html.haml b/app/views/import/bitbucket/status.html.haml index f8b4b107513..ac09b71ae89 100644 --- a/app/views/import/bitbucket/status.html.haml +++ b/app/views/import/bitbucket/status.html.haml @@ -1,5 +1,6 @@ -- page_title "Bitbucket import" -- header_title "Projects", root_path +- page_title 'Bitbucket import' +- header_title 'Projects', root_path + %h3.page-title %i.fa.fa-bitbucket Import projects from Bitbucket @@ -10,13 +11,13 @@ %hr %p - if @incompatible_repos.any? - = button_tag class: "btn btn-import btn-success js-import-all" do + = button_tag class: 'btn btn-import btn-success js-import-all' do Import all compatible projects - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') - else - = button_tag class: "btn btn-success js-import-all" do + = button_tag class: 'btn btn-import btn-success js-import-all' do Import all projects - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') .table-responsive %table.table.import-jobs @@ -32,7 +33,7 @@ - @already_added_projects.each do |project| %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"} %td - = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: "_blank" + = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: '_blank' %td = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project] %td.job-status @@ -47,31 +48,41 @@ = project.human_import_status_name - @repos.each do |repo| - %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"} + %tr{id: "repo_#{repo.owner}___#{repo.slug}"} %td - = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank" + = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: "_blank" %td.import-target - = import_project_target(repo['owner'], repo['slug']) + %fieldset.row + .input-group + .project-path.input-group-btn + - if current_user.can_select_namespace? + - selected = params[:namespace_id] || :current_user + - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner, path: repo.owner) } : {} + = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 } + - else + = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true + %span.input-group-addon / + = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true %td.import-actions.job-status - = button_tag class: "btn btn-import js-add-to-import" do + = button_tag class: 'btn btn-import js-add-to-import' do Import - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') - @incompatible_repos.each do |repo| - %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"} + %tr{id: "repo_#{repo.owner}___#{repo.slug}"} %td - = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank" + = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: '_blank' %td.import-target %td.import-actions-job-status - = label_tag "Incompatible Project", nil, class: "label label-danger" + = label_tag 'Incompatible Project', nil, class: 'label label-danger' - if @incompatible_repos.any? %p One or more of your Bitbucket projects cannot be imported into GitLab directly because they use Subversion or Mercurial for version control, rather than Git. Please convert - = link_to "them to Git,", "https://www.atlassian.com/git/tutorials/migrating-overview" + = link_to 'them to Git,', 'https://www.atlassian.com/git/tutorials/migrating-overview' and go through the - = link_to "import flow", status_import_bitbucket_path, "data-no-turbolink" => "true" + = link_to 'import flow', status_import_bitbucket_path, 'data-no-turbolink' => 'true' again. .js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_path}", import_path: "#{import_bitbucket_path}" } } diff --git a/changelogs/unreleased/bitbucket-oauth2.yml b/changelogs/unreleased/bitbucket-oauth2.yml new file mode 100644 index 00000000000..97d82518b7b --- /dev/null +++ b/changelogs/unreleased/bitbucket-oauth2.yml @@ -0,0 +1,4 @@ +--- +title: Refactor Bitbucket importer to use BitBucket API Version 2 +merge_request: +author: diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 26c30e523a7..ab5a0561b8c 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -26,3 +26,9 @@ if Gitlab.config.omniauth.enabled end end end + +module OmniAuth + module Strategies + autoload :Bitbucket, Rails.root.join('lib', 'omniauth', 'strategies', 'bitbucket') + end +end diff --git a/config/initializers/public_key.rb b/config/initializers/public_key.rb deleted file mode 100644 index e4f09a2d020..00000000000 --- a/config/initializers/public_key.rb +++ /dev/null @@ -1,2 +0,0 @@ -path = File.expand_path("~/.ssh/bitbucket_rsa.pub") -Gitlab::BitbucketImport.public_key = File.read(path) if File.exist?(path) diff --git a/doc/README.md b/doc/README.md index eba1e9845b1..a60a5359540 100644 --- a/doc/README.md +++ b/doc/README.md @@ -8,7 +8,7 @@ - [GitLab as OAuth2 authentication service provider](integration/oauth_provider.md). It allows you to login to other applications from GitLab. - [Container Registry](user/project/container_registry.md) Learn how to use GitLab Container Registry. - [GitLab basics](gitlab-basics/README.md) Find step by step how to start working on your commandline and on GitLab. -- [Importing to GitLab](workflow/importing/README.md). +- [Importing to GitLab](workflow/importing/README.md) Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab. - [Importing and exporting projects between instances](user/project/settings/import_export.md). - [Markdown](user/markdown.md) GitLab's advanced formatting system. - [Migrating from SVN](workflow/importing/migrating_from_svn.md) Convert a SVN repository to Git and GitLab. diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 9122dc62e39..5df6e103f42 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -18,8 +18,10 @@ Bitbucket.org. ## Bitbucket OmniAuth provider > **Note:** -Make sure to first follow the [Initial OmniAuth configuration][init-oauth] -before proceeding with setting up the Bitbucket integration. +GitLab 8.15 significantly simplified the way to integrate Bitbucket.org with +GitLab. You are encouraged to upgrade your GitLab instance if you haven't done +already. If you're using GitLab 8.14 and below, [use the previous integration +docs][bb-old]. To enable the Bitbucket OmniAuth provider you must register your application with Bitbucket.org. Bitbucket will generate an application ID and secret key for @@ -44,14 +46,12 @@ you to use. And grant at least the following permissions: ``` - Account: Email - Repositories: Read, Admin + Account: Email, Read + Repositories: Read + Pull Requests: Read + Issues: Read ``` - >**Note:** - It may seem a little odd to giving GitLab admin permissions to repositories, - but this is needed in order for GitLab to be able to clone the repositories. - ![Bitbucket OAuth settings page](img/bitbucket_oauth_settings_page.png) 1. Select **Save**. @@ -93,7 +93,8 @@ you to use. ```yaml - { name: 'bitbucket', app_id: 'BITBUCKET_APP_KEY', - app_secret: 'BITBUCKET_APP_SECRET' } + app_secret: 'BITBUCKET_APP_SECRET', + url: 'https://bitbucket.org/' } ``` --- @@ -112,100 +113,12 @@ well, the user will be returned to GitLab and will be signed in. ## Bitbucket project import -To allow projects to be imported directly into GitLab, Bitbucket requires two -extra setup steps compared to [GitHub](github.md) and [GitLab.com](gitlab.md). - -Bitbucket doesn't allow OAuth applications to clone repositories over HTTPS, and -instead requires GitLab to use SSH and identify itself using your GitLab -server's SSH key. - -To be able to access repositories on Bitbucket, GitLab will automatically -register your public key with Bitbucket as a deploy key for the repositories to -be imported. Your public key needs to be at `~/.ssh/bitbucket_rsa` which -translates to `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages and to -`/home/git/.ssh/bitbucket_rsa` for installations from source. - ---- - -Below are the steps that will allow GitLab to be able to import your projects -from Bitbucket. - -1. Make sure you [have enabled the Bitbucket OAuth support](#bitbucket-omniauth-provider). -1. Create a new SSH key with an **empty passphrase**: - - ```sh - sudo -u git -H ssh-keygen - ``` - - When asked to 'Enter file in which to save the key' enter: - `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages or - `/home/git/.ssh/bitbucket_rsa` for installations from source. The name is - important so make sure to get it right. - - > **Warning:** - This key must NOT be associated with ANY existing Bitbucket accounts. If it - is, the import will fail with an `Access denied! Please verify you can add - deploy keys to this repository.` error. - -1. Next, you need to to configure the SSH client to use your new key. Open the - SSH configuration file of the `git` user: - - ``` - # For Omnibus packages - sudo editor /var/opt/gitlab/.ssh/config - - # For installations from source - sudo editor /home/git/.ssh/config - ``` - -1. Add a host configuration for `bitbucket.org`: - - ```sh - Host bitbucket.org - IdentityFile ~/.ssh/bitbucket_rsa - User git - ``` - -1. Save the file and exit. -1. Manually connect to `bitbucket.org` over SSH, while logged in as the `git` - user that GitLab will use: - - ```sh - sudo -u git -H ssh bitbucket.org - ``` - - That step is performed because GitLab needs to connect to Bitbucket over SSH, - in order to add `bitbucket.org` to your GitLab server's known SSH hosts. - -1. Verify the RSA key fingerprint you'll see in the response matches the one - in the [Bitbucket documentation][bitbucket-docs] (the specific IP address - doesn't matter): - - ```sh - The authenticity of host 'bitbucket.org (104.192.143.1)' can't be established. - RSA key fingerprint is SHA256:zzXQOXSRBEiUtuE8AikJYKwbHaxvSc0ojez9YXaGp1A. - Are you sure you want to continue connecting (yes/no)? - ``` - -1. If the fingerprint matches, type `yes` to continue connecting and have - `bitbucket.org` be added to your known SSH hosts. After confirming you should - see a permission denied message. If you see an authentication successful - message you have done something wrong. The key you are using has already been - added to a Bitbucket account and will cause the import script to fail. Ensure - the key you are using CANNOT authenticate with Bitbucket. -1. Restart GitLab to allow it to find the new public key. - -Your GitLab server is now able to connect to Bitbucket over SSH. You should be -able to see the "Import projects from Bitbucket" option on the New Project page -enabled. - -## Acknowledgements - -Special thanks to the writer behind the following article: - -- http://stratus3d.com/blog/2015/09/06/migrating-from-bitbucket-to-local-gitlab-server/ +Once the above configuration is set up, you can use Bitbucket to sign into +GitLab and [start importing your projects][bb-import]. [init-oauth]: omniauth.md#initial-omniauth-configuration +[bb-import]: ../workflow/importing/import_projects_from_bitbucket.md +[bb-old]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-14-stable/doc/integration/bitbucket.md [bitbucket-docs]: https://confluence.atlassian.com/bitbucket/use-the-ssh-protocol-with-bitbucket-cloud-221449711.html#UsetheSSHprotocolwithBitbucketCloud-KnownhostorBitbucket%27spublickeyfingerprints [reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure [restart GitLab]: ../administration/restart_gitlab.md#installations-from-source diff --git a/doc/integration/img/bitbucket_oauth_settings_page.png b/doc/integration/img/bitbucket_oauth_settings_page.png Binary files differindex 8dbee9762d7..21ce82a6074 100644 --- a/doc/integration/img/bitbucket_oauth_settings_page.png +++ b/doc/integration/img/bitbucket_oauth_settings_page.png diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png Binary files differdeleted file mode 100644 index df55a081803..00000000000 --- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png +++ /dev/null diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png Binary files differdeleted file mode 100644 index 5253889d251..00000000000 --- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png +++ /dev/null diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png Binary files differdeleted file mode 100644 index ffa87ce5b2e..00000000000 --- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png +++ /dev/null diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png Binary files differdeleted file mode 100644 index 1a5661de75d..00000000000 --- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png +++ /dev/null diff --git a/doc/workflow/importing/img/bitbucket_import_grant_access.png b/doc/workflow/importing/img/bitbucket_import_grant_access.png Binary files differnew file mode 100644 index 00000000000..429904e621d --- /dev/null +++ b/doc/workflow/importing/img/bitbucket_import_grant_access.png diff --git a/doc/workflow/importing/img/bitbucket_import_new_project.png b/doc/workflow/importing/img/bitbucket_import_new_project.png Binary files differnew file mode 100644 index 00000000000..8ed528c2f09 --- /dev/null +++ b/doc/workflow/importing/img/bitbucket_import_new_project.png diff --git a/doc/workflow/importing/img/bitbucket_import_select_project.png b/doc/workflow/importing/img/bitbucket_import_select_project.png Binary files differnew file mode 100644 index 00000000000..1bca6166ec8 --- /dev/null +++ b/doc/workflow/importing/img/bitbucket_import_select_project.png diff --git a/doc/workflow/importing/img/import_projects_from_github_new_project_page.png b/doc/workflow/importing/img/import_projects_from_github_new_project_page.png Binary files differdeleted file mode 100644 index b23ade4480c..00000000000 --- a/doc/workflow/importing/img/import_projects_from_github_new_project_page.png +++ /dev/null diff --git a/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png b/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png Binary files differindex f50d9266991..1ccb38a815e 100644 --- a/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png +++ b/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png diff --git a/doc/workflow/importing/img/import_projects_from_new_project_page.png b/doc/workflow/importing/img/import_projects_from_new_project_page.png Binary files differnew file mode 100644 index 00000000000..97ca30b2087 --- /dev/null +++ b/doc/workflow/importing/img/import_projects_from_new_project_page.png diff --git a/doc/workflow/importing/import_projects_from_bitbucket.md b/doc/workflow/importing/import_projects_from_bitbucket.md index 520c4216295..b6d47e5afa2 100644 --- a/doc/workflow/importing/import_projects_from_bitbucket.md +++ b/doc/workflow/importing/import_projects_from_bitbucket.md @@ -1,26 +1,61 @@ # Import your project from Bitbucket to GitLab
-It takes just a few steps to import your existing Bitbucket projects to GitLab. But keep in mind that it is possible only if Bitbucket support is enabled on your GitLab instance. You can read more about Bitbucket support [here](../../integration/bitbucket.md).
+Import your projects from Bitbucket to GitLab with minimal effort.
-* Sign in to GitLab.com and go to your dashboard
+## Overview
-* Click on "New project"
+>**Note:**
+The [Bitbucket integration][bb-import] must be first enabled in order to be
+able to import your projects from Bitbucket. Ask your GitLab administrator
+to enable this if not already.
-![New project in GitLab](bitbucket_importer/bitbucket_import_new_project.png)
+- At its current state, the Bitbucket importer can import:
+ - the repository description (GitLab 7.7+)
+ - the Git repository data (GitLab 7.7+)
+ - the issues (GitLab 7.7+)
+ - the issue comments (GitLab 8.15+)
+ - the pull requests (GitLab 8.4+)
+ - the pull request comments (GitLab 8.15+)
+ - the milestones (GitLab 8.15+)
+- References to pull requests and issues are preserved (GitLab 8.7+)
+- Repository public access is retained. If a repository is private in Bitbucket
+ it will be created as private in GitLab as well.
-* Click on the "Bitbucket" button
-![Bitbucket](bitbucket_importer/bitbucket_import_select_bitbucket.png)
+## How it works
-* Grant GitLab access to your Bitbucket account
+When issues/pull requests are being imported, the Bitbucket importer tries to find
+the Bitbucket author/assignee in GitLab's database using the Bitbucket ID. For this
+to work, the Bitbucket author/assignee should have signed in beforehand in GitLab
+and [**associated their Bitbucket account**][social sign-in]. If the user is not
+found in GitLab's database, the project creator (most of the times the current
+user that started the import process) is set as the author, but a reference on
+the issue about the original Bitbucket author is kept.
-![Grant access](bitbucket_importer/bitbucket_import_grant_access.png)
+The importer will create any new namespaces (groups) if they don't exist or in
+the case the namespace is taken, the repository will be imported under the user's
+namespace that started the import process.
-* Click on the projects that you'd like to import or "Import all projects"
+## Importing your Bitbucket repositories
-![Import projects](bitbucket_importer/bitbucket_import_select_project.png)
+1. Sign in to GitLab and go to your dashboard.
+1. Click on **New project**.
-A new GitLab project will be created with your imported data.
+ ![New project in GitLab](img/bitbucket_import_new_project.png)
-### Note
-Milestones and wiki pages are not imported from Bitbucket.
+1. Click on the "Bitbucket" button
+
+ ![Bitbucket](img/import_projects_from_new_project_page.png)
+
+1. Grant GitLab access to your Bitbucket account
+
+ ![Grant access](img/bitbucket_import_grant_access.png)
+
+1. Click on the projects that you'd like to import or **Import all projects**.
+ You can also select the namespace under which each project will be
+ imported.
+
+ ![Import projects](img/bitbucket_import_select_project.png)
+
+[bb-import]: ../../integration/bitbucket.md
+[social sign-in]: ../../user/profile/account/social_sign_in.md
diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md index c36dfdb78ec..b3660aa8030 100644 --- a/doc/workflow/importing/import_projects_from_github.md +++ b/doc/workflow/importing/import_projects_from_github.md @@ -40,7 +40,7 @@ namespace that started the import process. The importer page is visible when you create a new project.
-![New project page on GitLab](img/import_projects_from_github_new_project_page.png)
+![New project page on GitLab](img/import_projects_from_new_project_page.png)
Click on the **GitHub** link and the import authorization process will start.
There are two ways to authorize access to your GitHub repositories:
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..7e55cf4deab --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..8969ecd1c19 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,67 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..7d2f92d577a 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,234 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) end - def identifier - project.import_source + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def has_issues? - client.project(identifier)["has_issues"] + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) + end + + def repo + @repo ||= client.repo(project.import_source) end def import_issues - issues = client.issues(identifier) + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } + end + end + end + + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) - comments = client.issue_comments(identifier, issue["local_id"]) + inline_comments, pr_comments = comments.partition(&:inline?) - if comments.any? - body += @formatter.comments_header - end + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) + + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..eb03882ab26 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,15 +14,15 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data } ).execute end end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/spec/controllers/import/bitbucket_controller_spec.rb b/spec/controllers/import/bitbucket_controller_spec.rb index 1d3c9fbbe2f..ce7c0b334ee 100644 --- a/spec/controllers/import/bitbucket_controller_spec.rb +++ b/spec/controllers/import/bitbucket_controller_spec.rb @@ -6,11 +6,11 @@ describe Import::BitbucketController do let(:user) { create(:user) } let(:token) { "asdasd12345" } let(:secret) { "sekrettt" } - let(:access_params) { { bitbucket_access_token: token, bitbucket_access_token_secret: secret } } + let(:refresh_token) { SecureRandom.hex(15) } + let(:access_params) { { token: token, expires_at: nil, expires_in: nil, refresh_token: nil } } def assign_session_tokens - session[:bitbucket_access_token] = token - session[:bitbucket_access_token_secret] = secret + session[:bitbucket_token] = token end before do @@ -24,29 +24,36 @@ describe Import::BitbucketController do end it "updates access token" do - access_token = double(token: token, secret: secret) - allow_any_instance_of(Gitlab::BitbucketImport::Client). + expires_at = Time.now + 1.day + expires_in = 1.day + access_token = double(token: token, + secret: secret, + expires_at: expires_at, + expires_in: expires_in, + refresh_token: refresh_token) + allow_any_instance_of(OAuth2::Client). to receive(:get_token).and_return(access_token) stub_omniauth_provider('bitbucket') get :callback - expect(session[:bitbucket_access_token]).to eq(token) - expect(session[:bitbucket_access_token_secret]).to eq(secret) + expect(session[:bitbucket_token]).to eq(token) + expect(session[:bitbucket_refresh_token]).to eq(refresh_token) + expect(session[:bitbucket_expires_at]).to eq(expires_at) + expect(session[:bitbucket_expires_in]).to eq(expires_in) expect(controller).to redirect_to(status_import_bitbucket_url) end end describe "GET status" do before do - @repo = OpenStruct.new(slug: 'vim', owner: 'asd') + @repo = double(slug: 'vim', owner: 'asd', full_name: 'asd/vim', "valid?" => true) assign_session_tokens end it "assigns variables" do @project = create(:project, import_type: 'bitbucket', creator_id: user.id) - client = stub_client(projects: [@repo]) - allow(client).to receive(:incompatible_projects).and_return([]) + allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo]) get :status @@ -57,7 +64,7 @@ describe Import::BitbucketController do it "does not show already added project" do @project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim') - stub_client(projects: [@repo]) + allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo]) get :status @@ -70,19 +77,16 @@ describe Import::BitbucketController do let(:bitbucket_username) { user.username } let(:bitbucket_user) do - { user: { username: bitbucket_username } }.with_indifferent_access + double(username: bitbucket_username) end let(:bitbucket_repo) do - { slug: "vim", owner: bitbucket_username }.with_indifferent_access + double(slug: "vim", owner: bitbucket_username, name: 'vim') end before do - allow(Gitlab::BitbucketImport::KeyAdder). - to receive(:new).with(bitbucket_repo, user, access_params). - and_return(double(execute: true)) - - stub_client(user: bitbucket_user, project: bitbucket_repo) + allow_any_instance_of(Bitbucket::Client).to receive(:repo).and_return(bitbucket_repo) + allow_any_instance_of(Bitbucket::Client).to receive(:user).and_return(bitbucket_user) assign_session_tokens end @@ -90,7 +94,7 @@ describe Import::BitbucketController do context "when the Bitbucket user and GitLab user's usernames match" do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -102,7 +106,7 @@ describe Import::BitbucketController do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -114,7 +118,7 @@ describe Import::BitbucketController do let(:other_username) { "someone_else" } before do - bitbucket_repo["owner"] = other_username + allow(bitbucket_repo).to receive(:owner).and_return(other_username) end context "when a namespace with the Bitbucket user's username already exists" do @@ -123,7 +127,7 @@ describe Import::BitbucketController do context "when the namespace is owned by the GitLab user" do it "takes the existing namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, existing_namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, existing_namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -156,7 +160,7 @@ describe Import::BitbucketController do it "takes the new namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, an_instance_of(Group), user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, an_instance_of(Group), user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -177,7 +181,7 @@ describe Import::BitbucketController do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js diff --git a/spec/lib/bitbucket/collection_spec.rb b/spec/lib/bitbucket/collection_spec.rb new file mode 100644 index 00000000000..015a7f80e03 --- /dev/null +++ b/spec/lib/bitbucket/collection_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +# Emulates paginator. It returns 2 pages with results +class TestPaginator + def initialize + @current_page = 0 + end + + def items + @current_page += 1 + + raise StopIteration if @current_page > 2 + + ["result_1_page_#{@current_page}", "result_2_page_#{@current_page}"] + end +end + +describe Bitbucket::Collection do + it "iterates paginator" do + collection = described_class.new(TestPaginator.new) + + expect(collection.to_a).to match(["result_1_page_1", "result_2_page_1", "result_1_page_2", "result_2_page_2"]) + end +end diff --git a/spec/lib/bitbucket/connection_spec.rb b/spec/lib/bitbucket/connection_spec.rb new file mode 100644 index 00000000000..14faeb231a9 --- /dev/null +++ b/spec/lib/bitbucket/connection_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe Bitbucket::Connection do + before do + allow_any_instance_of(described_class).to receive(:provider).and_return(double(app_id: '', app_secret: '')) + end + + describe '#get' do + it 'calls OAuth2::AccessToken::get' do + expect_any_instance_of(OAuth2::AccessToken).to receive(:get).and_return(double(parsed: true)) + + connection = described_class.new({}) + + connection.get('/users') + end + end + + describe '#expired?' do + it 'calls connection.expired?' do + expect_any_instance_of(OAuth2::AccessToken).to receive(:expired?).and_return(true) + + expect(described_class.new({}).expired?).to be_truthy + end + end + + describe '#refresh!' do + it 'calls connection.refresh!' do + response = double(token: nil, expires_at: nil, expires_in: nil, refresh_token: nil) + + expect_any_instance_of(OAuth2::AccessToken).to receive(:refresh!).and_return(response) + + described_class.new({}).refresh! + end + end +end diff --git a/spec/lib/bitbucket/page_spec.rb b/spec/lib/bitbucket/page_spec.rb new file mode 100644 index 00000000000..04d5a0470b1 --- /dev/null +++ b/spec/lib/bitbucket/page_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe Bitbucket::Page do + let(:response) { { 'values' => [{ 'username' => 'Ben' }], 'pagelen' => 2, 'next' => '' } } + + before do + # Autoloading hack + Bitbucket::Representation::User.new({}) + end + + describe '#items' do + it 'returns collection of needed objects' do + page = described_class.new(response, :user) + + expect(page.items.first).to be_a(Bitbucket::Representation::User) + expect(page.items.count).to eq(1) + end + end + + describe '#attrs' do + it 'returns attributes' do + page = described_class.new(response, :user) + + expect(page.attrs.keys).to include(:pagelen, :next) + end + end + + describe '#next?' do + it 'returns true' do + page = described_class.new(response, :user) + + expect(page.next?).to be_truthy + end + + it 'returns false' do + response['next'] = nil + page = described_class.new(response, :user) + + expect(page.next?).to be_falsey + end + end + + describe '#next' do + it 'returns next attribute' do + page = described_class.new(response, :user) + + expect(page.next).to eq('') + end + end +end diff --git a/spec/lib/bitbucket/paginator_spec.rb b/spec/lib/bitbucket/paginator_spec.rb new file mode 100644 index 00000000000..2c972da682e --- /dev/null +++ b/spec/lib/bitbucket/paginator_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe Bitbucket::Paginator do + let(:last_page) { double(:page, next?: false, items: ['item_2']) } + let(:first_page) { double(:page, next?: true, next: last_page, items: ['item_1']) } + + describe 'items' do + it 'return items and raises StopIteration in the end' do + paginator = described_class.new(nil, nil, nil) + + allow(paginator).to receive(:fetch_next_page).and_return(first_page) + expect(paginator.items).to match(['item_1']) + + allow(paginator).to receive(:fetch_next_page).and_return(last_page) + expect(paginator.items).to match(['item_2']) + + allow(paginator).to receive(:fetch_next_page).and_return(nil) + expect{ paginator.items }.to raise_error(StopIteration) + end + end +end diff --git a/spec/lib/bitbucket/representation/comment_spec.rb b/spec/lib/bitbucket/representation/comment_spec.rb new file mode 100644 index 00000000000..fec243a9f96 --- /dev/null +++ b/spec/lib/bitbucket/representation/comment_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Bitbucket::Representation::Comment do + describe '#author' do + it { expect(described_class.new('user' => { 'username' => 'Ben' }).author).to eq('Ben') } + it { expect(described_class.new({}).author).to be_nil } + end + + describe '#note' do + it { expect(described_class.new('content' => { 'raw' => 'Text' }).note).to eq('Text') } + it { expect(described_class.new({}).note).to be_nil } + end + + describe '#created_at' do + it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) } + end + + describe '#updated_at' do + it { expect(described_class.new('updated_on' => Date.today).updated_at).to eq(Date.today) } + it { expect(described_class.new('created_on' => Date.today).updated_at).to eq(Date.today) } + end +end diff --git a/spec/lib/bitbucket/representation/issue_spec.rb b/spec/lib/bitbucket/representation/issue_spec.rb new file mode 100644 index 00000000000..20f47224aa8 --- /dev/null +++ b/spec/lib/bitbucket/representation/issue_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +describe Bitbucket::Representation::Issue do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#kind' do + it { expect(described_class.new('kind' => 'bug').kind).to eq('bug') } + end + + describe '#milestone' do + it { expect(described_class.new({ 'milestone' => { 'name' => '1.0' } }).milestone).to eq('1.0') } + it { expect(described_class.new({}).milestone).to be_nil } + end + + describe '#author' do + it { expect(described_class.new({ 'reporter' => { 'username' => 'Ben' } }).author).to eq('Ben') } + it { expect(described_class.new({}).author).to be_nil } + end + + describe '#description' do + it { expect(described_class.new({ 'content' => { 'raw' => 'Text' } }).description).to eq('Text') } + it { expect(described_class.new({}).description).to be_nil } + end + + describe '#state' do + it { expect(described_class.new({ 'state' => 'invalid' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'wontfix' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'resolved' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'duplicate' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'closed' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'opened' }).state).to eq('opened') } + end + + describe '#title' do + it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') } + end + + describe '#created_at' do + it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) } + end + + describe '#updated_at' do + it { expect(described_class.new('edited_on' => Date.today).updated_at).to eq(Date.today) } + end +end diff --git a/spec/lib/bitbucket/representation/pull_request_comment_spec.rb b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb new file mode 100644 index 00000000000..673dcf22ce8 --- /dev/null +++ b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +describe Bitbucket::Representation::PullRequestComment do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#file_path' do + it { expect(described_class.new('inline' => { 'path' => '/path' }).file_path).to eq('/path') } + end + + describe '#old_pos' do + it { expect(described_class.new('inline' => { 'from' => 3 }).old_pos).to eq(3) } + end + + describe '#new_pos' do + it { expect(described_class.new('inline' => { 'to' => 3 }).new_pos).to eq(3) } + end + + describe '#parent_id' do + it { expect(described_class.new({ 'parent' => { 'id' => 2 } }).parent_id).to eq(2) } + it { expect(described_class.new({}).parent_id).to be_nil } + end + + describe '#inline?' do + it { expect(described_class.new('inline' => {}).inline?).to be_truthy } + it { expect(described_class.new({}).inline?).to be_falsey } + end + + describe '#has_parent?' do + it { expect(described_class.new('parent' => {}).has_parent?).to be_truthy } + it { expect(described_class.new({}).has_parent?).to be_falsey } + end +end diff --git a/spec/lib/bitbucket/representation/pull_request_spec.rb b/spec/lib/bitbucket/representation/pull_request_spec.rb new file mode 100644 index 00000000000..30453528be4 --- /dev/null +++ b/spec/lib/bitbucket/representation/pull_request_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +describe Bitbucket::Representation::PullRequest do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#author' do + it { expect(described_class.new({ 'author' => { 'username' => 'Ben' } }).author).to eq('Ben') } + it { expect(described_class.new({}).author).to be_nil } + end + + describe '#description' do + it { expect(described_class.new({ 'description' => 'Text' }).description).to eq('Text') } + it { expect(described_class.new({}).description).to be_nil } + end + + describe '#state' do + it { expect(described_class.new({ 'state' => 'MERGED' }).state).to eq('merged') } + it { expect(described_class.new({ 'state' => 'DECLINED' }).state).to eq('closed') } + it { expect(described_class.new({}).state).to eq('opened') } + end + + describe '#title' do + it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') } + end + + describe '#source_branch_name' do + it { expect(described_class.new({ source: { branch: { name: 'feature' } } }.with_indifferent_access).source_branch_name).to eq('feature') } + it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_name).to be_nil } + end + + describe '#source_branch_sha' do + it { expect(described_class.new({ source: { commit: { hash: 'abcd123' } } }.with_indifferent_access).source_branch_sha).to eq('abcd123') } + it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_sha).to be_nil } + end + + describe '#target_branch_name' do + it { expect(described_class.new({ destination: { branch: { name: 'master' } } }.with_indifferent_access).target_branch_name).to eq('master') } + it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_name).to be_nil } + end + + describe '#target_branch_sha' do + it { expect(described_class.new({ destination: { commit: { hash: 'abcd123' } } }.with_indifferent_access).target_branch_sha).to eq('abcd123') } + it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_sha).to be_nil } + end +end diff --git a/spec/lib/bitbucket/representation/user_spec.rb b/spec/lib/bitbucket/representation/user_spec.rb new file mode 100644 index 00000000000..f79ff4edb7b --- /dev/null +++ b/spec/lib/bitbucket/representation/user_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe Bitbucket::Representation::User do + describe '#username' do + it 'returns correct value' do + user = described_class.new('username' => 'Ben') + + expect(user.username).to eq('Ben') + end + end +end diff --git a/spec/lib/gitlab/bitbucket_import/client_spec.rb b/spec/lib/gitlab/bitbucket_import/client_spec.rb deleted file mode 100644 index 7543c29bcc4..00000000000 --- a/spec/lib/gitlab/bitbucket_import/client_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'spec_helper' - -describe Gitlab::BitbucketImport::Client, lib: true do - include ImportSpecHelper - - let(:token) { '123456' } - let(:secret) { 'secret' } - let(:client) { Gitlab::BitbucketImport::Client.new(token, secret) } - - before do - stub_omniauth_provider('bitbucket') - end - - it 'all OAuth client options are symbols' do - client.consumer.options.keys.each do |key| - expect(key).to be_kind_of(Symbol) - end - end - - context 'issues' do - let(:per_page) { 50 } - let(:count) { 95 } - let(:sample_issues) do - issues = [] - - count.times do |i| - issues << { local_id: i } - end - - issues - end - let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } } - let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } } - let(:project_id) { 'namespace/repo' } - - it 'retrieves issues over a number of pages' do - stub_request(:get, - "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0"). - to_return(status: 200, - body: first_sample_data.to_json, - headers: {}) - - stub_request(:get, - "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50"). - to_return(status: 200, - body: second_sample_data.to_json, - headers: {}) - - issues = client.issues(project_id) - expect(issues.count).to eq(95) - end - end - - context 'project import' do - it 'calls .from_project with no errors' do - project = create(:empty_project) - project.import_url = "ssh://git@bitbucket.org/test/test.git" - project.create_or_update_import_data(credentials: - { user: "git", - password: nil, - bb_session: { bitbucket_access_token: "test", - bitbucket_access_token_secret: "test" } }) - - expect { described_class.from_project(project) }.not_to raise_error - end - end -end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index aa00f32becb..53f3c73ade4 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -18,15 +18,21 @@ describe Gitlab::BitbucketImport::Importer, lib: true do "closed" # undocumented status ] end + let(:sample_issues_statuses) do issues = [] statuses.map.with_index do |status, index| issues << { - local_id: index, - status: status, + id: index, + state: status, title: "Issue #{index}", - content: "Some content to issue #{index}" + kind: 'bug', + content: { + raw: "Some content to issue #{index}", + markup: "markdown", + html: "Some content to issue #{index}" + } } end @@ -34,14 +40,16 @@ describe Gitlab::BitbucketImport::Importer, lib: true do end let(:project_identifier) { 'namespace/repo' } + let(:data) do { 'bb_session' => { - 'bitbucket_access_token' => "123456", - 'bitbucket_access_token_secret' => "secret" + 'bitbucket_token' => "123456", + 'bitbucket_refresh_token' => "secret" } } end + let(:project) do create( :project, @@ -49,11 +57,13 @@ describe Gitlab::BitbucketImport::Importer, lib: true do import_data: ProjectImportData.new(credentials: data) ) end + let(:importer) { Gitlab::BitbucketImport::Importer.new(project) } + let(:issues_statuses_sample_data) do { count: sample_issues_statuses.count, - issues: sample_issues_statuses + values: sample_issues_statuses } end @@ -61,26 +71,46 @@ describe Gitlab::BitbucketImport::Importer, lib: true do before do stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}" - ).to_return(status: 200, body: { has_issues: true }.to_json) + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: { has_issues: true, full_name: project_identifier }.to_json) stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues?limit=50&sort=utc_created_on&start=0" - ).to_return(status: 200, body: issues_statuses_sample_data.to_json) + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=50&sort=created_on" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: issues_statuses_sample_data.to_json) + + stub_request(:get, "https://api.bitbucket.org/2.0/repositories/namespace/repo?pagelen=50&sort=created_on"). + with(headers: { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization' => 'Bearer', 'User-Agent' => 'Faraday v0.9.2' }). + to_return(status: 200, + body: "", + headers: {}) sample_issues_statuses.each_with_index do |issue, index| stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues/#{issue[:local_id]}/comments" + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues/#{issue[:id]}/comments?pagelen=50&sort=created_on" ).to_return( status: 200, - body: [{ author_info: { username: "username" }, utc_created_on: index }].to_json + headers: { "Content-Type" => "application/json" }, + body: { author_info: { username: "username" }, utc_created_on: index }.to_json ) end + + stub_request( + :get, + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/pullrequests?pagelen=50&sort=created_on&state=ALL" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: {}.to_json) end it 'map statuses to open or closed' do + # HACK: Bitbucket::Representation.const_get('Issue') seems to return ::Issue without this + Bitbucket::Representation::Issue.new({}) importer.execute expect(project.issues.where(state: "closed").size).to eq(5) diff --git a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb index e1c60e07b4d..b6d052a4612 100644 --- a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb @@ -2,14 +2,18 @@ require 'spec_helper' describe Gitlab::BitbucketImport::ProjectCreator, lib: true do let(:user) { create(:user) } + let(:repo) do - { - name: 'Vim', - slug: 'vim', - is_private: true, - owner: "asd" - }.with_indifferent_access + double(name: 'Vim', + slug: 'vim', + description: 'Test repo', + is_private: true, + owner: "asd", + full_name: 'Vim repo', + visibility_level: Gitlab::VisibilityLevel::PRIVATE, + clone_url: 'ssh://git@bitbucket.org/asd/vim.git') end + let(:namespace){ create(:group, owner: user) } let(:token) { "asdasd12345" } let(:secret) { "sekrettt" } @@ -22,7 +26,7 @@ describe Gitlab::BitbucketImport::ProjectCreator, lib: true do it 'creates project' do allow_any_instance_of(Project).to receive(:add_import_job) - project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user, access_params) + project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, 'vim', namespace, user, access_params) project = project_creator.execute expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git") |