summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDouglas Barbosa Alexandre <dbalexandre@gmail.com>2016-12-16 23:25:14 +0000
committerDouglas Barbosa Alexandre <dbalexandre@gmail.com>2016-12-16 23:25:14 +0000
commita22be3065dc9c80d3a18ca81eda480b9c39e35a6 (patch)
treed29fc79464dbf3f1fd09d034a4b73c8eb230a9e3
parent12a7e717d7b9fdd265d54a9c5bd07394e304b187 (diff)
parenta3be4aeb7a71cc940394a5f13d09e79fcafdb1d5 (diff)
downloadgitlab-ce-a22be3065dc9c80d3a18ca81eda480b9c39e35a6.tar.gz
Merge branch 'bitbucket-oauth2' into 'master'
Refactor Bitbucket importer to use BitBucket API Version 2 ## What does this MR do? ## Are there points in the code the reviewer needs to double check? ## Why was this MR needed? ## What are the relevant issue numbers? https://gitlab.com/gitlab-org/gitlab-ce/issues/19946 ## Screenshots (if relevant) This MR needs the following permissions in the Bitbucket OAuth settings: ![image](/uploads/a26ae5e430a724bf581a92da7028ce3c/image.png) - [] ## Does this MR meet the acceptance criteria? - [ ] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added - [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md) - [ ] API support added - Tests - [ ] Added for this feature/bug - [ ] All builds are passing - [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides) - [ ] Branch has no merge conflicts with `master` (if you do - rebase it please) - [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits) See merge request !5995
-rw-r--r--Gemfile1
-rw-r--r--Gemfile.lock5
-rw-r--r--app/controllers/application_controller.rb2
-rw-r--r--app/controllers/import/bitbucket_controller.rb83
-rw-r--r--app/views/import/bitbucket/status.html.haml45
-rw-r--r--changelogs/unreleased/bitbucket-oauth2.yml4
-rw-r--r--config/initializers/omniauth.rb6
-rw-r--r--config/initializers/public_key.rb2
-rw-r--r--doc/README.md2
-rw-r--r--doc/integration/bitbucket.md115
-rw-r--r--doc/integration/img/bitbucket_oauth_settings_page.pngbin30081 -> 5607 bytes
-rw-r--r--doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.pngbin30083 -> 0 bytes
-rw-r--r--doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.pngbin16502 -> 0 bytes
-rw-r--r--doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.pngbin46606 -> 0 bytes
-rw-r--r--doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.pngbin15288 -> 0 bytes
-rw-r--r--doc/workflow/importing/img/bitbucket_import_grant_access.pngbin0 -> 7248 bytes
-rw-r--r--doc/workflow/importing/img/bitbucket_import_new_project.pngbin0 -> 1316 bytes
-rw-r--r--doc/workflow/importing/img/bitbucket_import_select_project.pngbin0 -> 8688 bytes
-rw-r--r--doc/workflow/importing/img/import_projects_from_github_new_project_page.pngbin11047 -> 0 bytes
-rw-r--r--doc/workflow/importing/img/import_projects_from_github_select_auth_method.pngbin17613 -> 17612 bytes
-rw-r--r--doc/workflow/importing/img/import_projects_from_new_project_page.pngbin0 -> 36821 bytes
-rw-r--r--doc/workflow/importing/import_projects_from_bitbucket.md61
-rw-r--r--doc/workflow/importing/import_projects_from_github.md2
-rw-r--r--lib/bitbucket/client.rb58
-rw-r--r--lib/bitbucket/collection.rb21
-rw-r--r--lib/bitbucket/connection.rb69
-rw-r--r--lib/bitbucket/error/unauthorized.rb6
-rw-r--r--lib/bitbucket/page.rb34
-rw-r--r--lib/bitbucket/paginator.rb36
-rw-r--r--lib/bitbucket/representation/base.rb17
-rw-r--r--lib/bitbucket/representation/comment.rb27
-rw-r--r--lib/bitbucket/representation/issue.rb53
-rw-r--r--lib/bitbucket/representation/pull_request.rb65
-rw-r--r--lib/bitbucket/representation/pull_request_comment.rb39
-rw-r--r--lib/bitbucket/representation/repo.rb67
-rw-r--r--lib/bitbucket/representation/user.rb9
-rw-r--r--lib/gitlab/bitbucket_import.rb6
-rw-r--r--lib/gitlab/bitbucket_import/client.rb142
-rw-r--r--lib/gitlab/bitbucket_import/importer.rb242
-rw-r--r--lib/gitlab/bitbucket_import/key_adder.rb24
-rw-r--r--lib/gitlab/bitbucket_import/key_deleter.rb23
-rw-r--r--lib/gitlab/bitbucket_import/project_creator.rb21
-rw-r--r--lib/omniauth/strategies/bitbucket.rb41
-rw-r--r--spec/controllers/import/bitbucket_controller_spec.rb52
-rw-r--r--spec/lib/bitbucket/collection_spec.rb24
-rw-r--r--spec/lib/bitbucket/connection_spec.rb35
-rw-r--r--spec/lib/bitbucket/page_spec.rb50
-rw-r--r--spec/lib/bitbucket/paginator_spec.rb21
-rw-r--r--spec/lib/bitbucket/representation/comment_spec.rb22
-rw-r--r--spec/lib/bitbucket/representation/issue_spec.rb47
-rw-r--r--spec/lib/bitbucket/representation/pull_request_comment_spec.rb34
-rw-r--r--spec/lib/bitbucket/representation/pull_request_spec.rb47
-rw-r--r--spec/lib/bitbucket/representation/user_spec.rb11
-rw-r--r--spec/lib/gitlab/bitbucket_import/client_spec.rb67
-rw-r--r--spec/lib/gitlab/bitbucket_import/importer_spec.rb54
-rw-r--r--spec/lib/gitlab/bitbucket_import/project_creator_spec.rb18
56 files changed, 1271 insertions, 539 deletions
diff --git a/Gemfile b/Gemfile
index 5eb8c32b168..bea31b53b1c 100644
--- a/Gemfile
+++ b/Gemfile
@@ -22,7 +22,6 @@ gem 'doorkeeper', '~> 4.2.0'
gem 'omniauth', '~> 1.3.1'
gem 'omniauth-auth0', '~> 1.4.1'
gem 'omniauth-azure-oauth2', '~> 0.0.6'
-gem 'omniauth-bitbucket', '~> 0.0.2'
gem 'omniauth-cas3', '~> 1.1.2'
gem 'omniauth-facebook', '~> 4.0.0'
gem 'omniauth-github', '~> 1.1.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index 23e45ddc16f..811adfc5c1d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -432,10 +432,6 @@ GEM
jwt (~> 1.0)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.1)
- omniauth-bitbucket (0.0.2)
- multi_json (~> 1.7)
- omniauth (~> 1.1)
- omniauth-oauth (~> 1.0)
omniauth-cas3 (1.1.3)
addressable (~> 2.3)
nokogiri (~> 1.6.6)
@@ -902,7 +898,6 @@ DEPENDENCIES
omniauth (~> 1.3.1)
omniauth-auth0 (~> 1.4.1)
omniauth-azure-oauth2 (~> 0.0.6)
- omniauth-bitbucket (~> 0.0.2)
omniauth-cas3 (~> 1.1.2)
omniauth-facebook (~> 4.0.0)
omniauth-github (~> 1.1.1)
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index bcc0b17bce2..4df80195ae1 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -262,7 +262,7 @@ class ApplicationController < ActionController::Base
end
def bitbucket_import_configured?
- Gitlab::OAuth::Provider.enabled?(:bitbucket) && Gitlab::BitbucketImport.public_key.present?
+ Gitlab::OAuth::Provider.enabled?(:bitbucket)
end
def google_code_import_enabled?
diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb
index 6ea54744da8..8e42cdf415f 100644
--- a/app/controllers/import/bitbucket_controller.rb
+++ b/app/controllers/import/bitbucket_controller.rb
@@ -2,50 +2,57 @@ class Import::BitbucketController < Import::BaseController
before_action :verify_bitbucket_import_enabled
before_action :bitbucket_auth, except: :callback
- rescue_from OAuth::Error, with: :bitbucket_unauthorized
- rescue_from Gitlab::BitbucketImport::Client::Unauthorized, with: :bitbucket_unauthorized
+ rescue_from OAuth2::Error, with: :bitbucket_unauthorized
+ rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized
def callback
- request_token = session.delete(:oauth_request_token)
- raise "Session expired!" if request_token.nil?
+ response = client.auth_code.get_token(params[:code], redirect_uri: callback_import_bitbucket_url)
- request_token.symbolize_keys!
-
- access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url)
-
- session[:bitbucket_access_token] = access_token.token
- session[:bitbucket_access_token_secret] = access_token.secret
+ session[:bitbucket_token] = response.token
+ session[:bitbucket_expires_at] = response.expires_at
+ session[:bitbucket_expires_in] = response.expires_in
+ session[:bitbucket_refresh_token] = response.refresh_token
redirect_to status_import_bitbucket_url
end
def status
- @repos = client.projects
- @incompatible_repos = client.incompatible_projects
+ bitbucket_client = Bitbucket::Client.new(credentials)
+ repos = bitbucket_client.repos
+
+ @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
- @already_added_projects = current_user.created_projects.where(import_type: "bitbucket")
+ @already_added_projects = current_user.created_projects.where(import_type: 'bitbucket')
already_added_projects_names = @already_added_projects.pluck(:import_source)
- @repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" }
+ @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) }
end
def jobs
- jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status])
- render json: jobs
+ render json: current_user.created_projects
+ .where(import_type: 'bitbucket')
+ .to_json(only: [:id, :import_status])
end
def create
+ bitbucket_client = Bitbucket::Client.new(credentials)
+
@repo_id = params[:repo_id].to_s
- repo = client.project(@repo_id.gsub('___', '/'))
- @project_name = repo['slug']
- @target_namespace = find_or_create_namespace(repo['owner'], client.user['user']['username'])
+ name = @repo_id.gsub('___', '/')
+ repo = bitbucket_client.repo(name)
+ @project_name = params[:new_name].presence || repo.name
- unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user, access_params).execute
- render 'deploy_key' and return
- end
+ repo_owner = repo.owner
+ repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
+ @target_namespace = params[:new_namespace].presence || repo_owner
+
+ namespace = find_or_create_namespace(@target_namespace, current_user)
- if current_user.can?(:create_projects, @target_namespace)
- @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @target_namespace, current_user, access_params).execute
+ if current_user.can?(:create_projects, namespace)
+ # The token in a session can be expired, we need to get most recent one because
+ # Bitbucket::Connection class refreshes it.
+ session[:bitbucket_token] = bitbucket_client.connection.token
+ @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @project_name, namespace, current_user, credentials).execute
else
render 'unauthorized'
end
@@ -54,8 +61,15 @@ class Import::BitbucketController < Import::BaseController
private
def client
- @client ||= Gitlab::BitbucketImport::Client.new(session[:bitbucket_access_token],
- session[:bitbucket_access_token_secret])
+ @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+ end
+
+ def provider
+ Gitlab::OAuth::Provider.config_for('bitbucket')
+ end
+
+ def options
+ OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
end
def verify_bitbucket_import_enabled
@@ -63,26 +77,23 @@ class Import::BitbucketController < Import::BaseController
end
def bitbucket_auth
- if session[:bitbucket_access_token].blank?
- go_to_bitbucket_for_permissions
- end
+ go_to_bitbucket_for_permissions if session[:bitbucket_token].blank?
end
def go_to_bitbucket_for_permissions
- request_token = client.request_token(callback_import_bitbucket_url)
- session[:oauth_request_token] = request_token
-
- redirect_to client.authorize_url(request_token, callback_import_bitbucket_url)
+ redirect_to client.auth_code.authorize_url(redirect_uri: callback_import_bitbucket_url)
end
def bitbucket_unauthorized
go_to_bitbucket_for_permissions
end
- def access_params
+ def credentials
{
- bitbucket_access_token: session[:bitbucket_access_token],
- bitbucket_access_token_secret: session[:bitbucket_access_token_secret]
+ token: session[:bitbucket_token],
+ expires_at: session[:bitbucket_expires_at],
+ expires_in: session[:bitbucket_expires_in],
+ refresh_token: session[:bitbucket_refresh_token]
}
end
end
diff --git a/app/views/import/bitbucket/status.html.haml b/app/views/import/bitbucket/status.html.haml
index f8b4b107513..ac09b71ae89 100644
--- a/app/views/import/bitbucket/status.html.haml
+++ b/app/views/import/bitbucket/status.html.haml
@@ -1,5 +1,6 @@
-- page_title "Bitbucket import"
-- header_title "Projects", root_path
+- page_title 'Bitbucket import'
+- header_title 'Projects', root_path
+
%h3.page-title
%i.fa.fa-bitbucket
Import projects from Bitbucket
@@ -10,13 +11,13 @@
%hr
%p
- if @incompatible_repos.any?
- = button_tag class: "btn btn-import btn-success js-import-all" do
+ = button_tag class: 'btn btn-import btn-success js-import-all' do
Import all compatible projects
- = icon("spinner spin", class: "loading-icon")
+ = icon('spinner spin', class: 'loading-icon')
- else
- = button_tag class: "btn btn-success js-import-all" do
+ = button_tag class: 'btn btn-import btn-success js-import-all' do
Import all projects
- = icon("spinner spin", class: "loading-icon")
+ = icon('spinner spin', class: 'loading-icon')
.table-responsive
%table.table.import-jobs
@@ -32,7 +33,7 @@
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td
- = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: "_blank"
+ = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: '_blank'
%td
= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
%td.job-status
@@ -47,31 +48,41 @@
= project.human_import_status_name
- @repos.each do |repo|
- %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
+ %tr{id: "repo_#{repo.owner}___#{repo.slug}"}
%td
- = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank"
+ = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: "_blank"
%td.import-target
- = import_project_target(repo['owner'], repo['slug'])
+ %fieldset.row
+ .input-group
+ .project-path.input-group-btn
+ - if current_user.can_select_namespace?
+ - selected = params[:namespace_id] || :current_user
+ - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner, path: repo.owner) } : {}
+ = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 }
+ - else
+ = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true
+ %span.input-group-addon /
+ = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true
%td.import-actions.job-status
- = button_tag class: "btn btn-import js-add-to-import" do
+ = button_tag class: 'btn btn-import js-add-to-import' do
Import
- = icon("spinner spin", class: "loading-icon")
+ = icon('spinner spin', class: 'loading-icon')
- @incompatible_repos.each do |repo|
- %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
+ %tr{id: "repo_#{repo.owner}___#{repo.slug}"}
%td
- = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank"
+ = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: '_blank'
%td.import-target
%td.import-actions-job-status
- = label_tag "Incompatible Project", nil, class: "label label-danger"
+ = label_tag 'Incompatible Project', nil, class: 'label label-danger'
- if @incompatible_repos.any?
%p
One or more of your Bitbucket projects cannot be imported into GitLab
directly because they use Subversion or Mercurial for version control,
rather than Git. Please convert
- = link_to "them to Git,", "https://www.atlassian.com/git/tutorials/migrating-overview"
+ = link_to 'them to Git,', 'https://www.atlassian.com/git/tutorials/migrating-overview'
and go through the
- = link_to "import flow", status_import_bitbucket_path, "data-no-turbolink" => "true"
+ = link_to 'import flow', status_import_bitbucket_path, 'data-no-turbolink' => 'true'
again.
.js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_path}", import_path: "#{import_bitbucket_path}" } }
diff --git a/changelogs/unreleased/bitbucket-oauth2.yml b/changelogs/unreleased/bitbucket-oauth2.yml
new file mode 100644
index 00000000000..97d82518b7b
--- /dev/null
+++ b/changelogs/unreleased/bitbucket-oauth2.yml
@@ -0,0 +1,4 @@
+---
+title: Refactor Bitbucket importer to use BitBucket API Version 2
+merge_request:
+author:
diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb
index 26c30e523a7..ab5a0561b8c 100644
--- a/config/initializers/omniauth.rb
+++ b/config/initializers/omniauth.rb
@@ -26,3 +26,9 @@ if Gitlab.config.omniauth.enabled
end
end
end
+
+module OmniAuth
+ module Strategies
+ autoload :Bitbucket, Rails.root.join('lib', 'omniauth', 'strategies', 'bitbucket')
+ end
+end
diff --git a/config/initializers/public_key.rb b/config/initializers/public_key.rb
deleted file mode 100644
index e4f09a2d020..00000000000
--- a/config/initializers/public_key.rb
+++ /dev/null
@@ -1,2 +0,0 @@
-path = File.expand_path("~/.ssh/bitbucket_rsa.pub")
-Gitlab::BitbucketImport.public_key = File.read(path) if File.exist?(path)
diff --git a/doc/README.md b/doc/README.md
index eba1e9845b1..a60a5359540 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -8,7 +8,7 @@
- [GitLab as OAuth2 authentication service provider](integration/oauth_provider.md). It allows you to login to other applications from GitLab.
- [Container Registry](user/project/container_registry.md) Learn how to use GitLab Container Registry.
- [GitLab basics](gitlab-basics/README.md) Find step by step how to start working on your commandline and on GitLab.
-- [Importing to GitLab](workflow/importing/README.md).
+- [Importing to GitLab](workflow/importing/README.md) Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab.
- [Importing and exporting projects between instances](user/project/settings/import_export.md).
- [Markdown](user/markdown.md) GitLab's advanced formatting system.
- [Migrating from SVN](workflow/importing/migrating_from_svn.md) Convert a SVN repository to Git and GitLab.
diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md
index 9122dc62e39..5df6e103f42 100644
--- a/doc/integration/bitbucket.md
+++ b/doc/integration/bitbucket.md
@@ -18,8 +18,10 @@ Bitbucket.org.
## Bitbucket OmniAuth provider
> **Note:**
-Make sure to first follow the [Initial OmniAuth configuration][init-oauth]
-before proceeding with setting up the Bitbucket integration.
+GitLab 8.15 significantly simplified the way to integrate Bitbucket.org with
+GitLab. You are encouraged to upgrade your GitLab instance if you haven't done
+already. If you're using GitLab 8.14 and below, [use the previous integration
+docs][bb-old].
To enable the Bitbucket OmniAuth provider you must register your application
with Bitbucket.org. Bitbucket will generate an application ID and secret key for
@@ -44,14 +46,12 @@ you to use.
And grant at least the following permissions:
```
- Account: Email
- Repositories: Read, Admin
+ Account: Email, Read
+ Repositories: Read
+ Pull Requests: Read
+ Issues: Read
```
- >**Note:**
- It may seem a little odd to giving GitLab admin permissions to repositories,
- but this is needed in order for GitLab to be able to clone the repositories.
-
![Bitbucket OAuth settings page](img/bitbucket_oauth_settings_page.png)
1. Select **Save**.
@@ -93,7 +93,8 @@ you to use.
```yaml
- { name: 'bitbucket',
app_id: 'BITBUCKET_APP_KEY',
- app_secret: 'BITBUCKET_APP_SECRET' }
+ app_secret: 'BITBUCKET_APP_SECRET',
+ url: 'https://bitbucket.org/' }
```
---
@@ -112,100 +113,12 @@ well, the user will be returned to GitLab and will be signed in.
## Bitbucket project import
-To allow projects to be imported directly into GitLab, Bitbucket requires two
-extra setup steps compared to [GitHub](github.md) and [GitLab.com](gitlab.md).
-
-Bitbucket doesn't allow OAuth applications to clone repositories over HTTPS, and
-instead requires GitLab to use SSH and identify itself using your GitLab
-server's SSH key.
-
-To be able to access repositories on Bitbucket, GitLab will automatically
-register your public key with Bitbucket as a deploy key for the repositories to
-be imported. Your public key needs to be at `~/.ssh/bitbucket_rsa` which
-translates to `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages and to
-`/home/git/.ssh/bitbucket_rsa` for installations from source.
-
----
-
-Below are the steps that will allow GitLab to be able to import your projects
-from Bitbucket.
-
-1. Make sure you [have enabled the Bitbucket OAuth support](#bitbucket-omniauth-provider).
-1. Create a new SSH key with an **empty passphrase**:
-
- ```sh
- sudo -u git -H ssh-keygen
- ```
-
- When asked to 'Enter file in which to save the key' enter:
- `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages or
- `/home/git/.ssh/bitbucket_rsa` for installations from source. The name is
- important so make sure to get it right.
-
- > **Warning:**
- This key must NOT be associated with ANY existing Bitbucket accounts. If it
- is, the import will fail with an `Access denied! Please verify you can add
- deploy keys to this repository.` error.
-
-1. Next, you need to to configure the SSH client to use your new key. Open the
- SSH configuration file of the `git` user:
-
- ```
- # For Omnibus packages
- sudo editor /var/opt/gitlab/.ssh/config
-
- # For installations from source
- sudo editor /home/git/.ssh/config
- ```
-
-1. Add a host configuration for `bitbucket.org`:
-
- ```sh
- Host bitbucket.org
- IdentityFile ~/.ssh/bitbucket_rsa
- User git
- ```
-
-1. Save the file and exit.
-1. Manually connect to `bitbucket.org` over SSH, while logged in as the `git`
- user that GitLab will use:
-
- ```sh
- sudo -u git -H ssh bitbucket.org
- ```
-
- That step is performed because GitLab needs to connect to Bitbucket over SSH,
- in order to add `bitbucket.org` to your GitLab server's known SSH hosts.
-
-1. Verify the RSA key fingerprint you'll see in the response matches the one
- in the [Bitbucket documentation][bitbucket-docs] (the specific IP address
- doesn't matter):
-
- ```sh
- The authenticity of host 'bitbucket.org (104.192.143.1)' can't be established.
- RSA key fingerprint is SHA256:zzXQOXSRBEiUtuE8AikJYKwbHaxvSc0ojez9YXaGp1A.
- Are you sure you want to continue connecting (yes/no)?
- ```
-
-1. If the fingerprint matches, type `yes` to continue connecting and have
- `bitbucket.org` be added to your known SSH hosts. After confirming you should
- see a permission denied message. If you see an authentication successful
- message you have done something wrong. The key you are using has already been
- added to a Bitbucket account and will cause the import script to fail. Ensure
- the key you are using CANNOT authenticate with Bitbucket.
-1. Restart GitLab to allow it to find the new public key.
-
-Your GitLab server is now able to connect to Bitbucket over SSH. You should be
-able to see the "Import projects from Bitbucket" option on the New Project page
-enabled.
-
-## Acknowledgements
-
-Special thanks to the writer behind the following article:
-
-- http://stratus3d.com/blog/2015/09/06/migrating-from-bitbucket-to-local-gitlab-server/
+Once the above configuration is set up, you can use Bitbucket to sign into
+GitLab and [start importing your projects][bb-import].
[init-oauth]: omniauth.md#initial-omniauth-configuration
+[bb-import]: ../workflow/importing/import_projects_from_bitbucket.md
+[bb-old]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-14-stable/doc/integration/bitbucket.md
[bitbucket-docs]: https://confluence.atlassian.com/bitbucket/use-the-ssh-protocol-with-bitbucket-cloud-221449711.html#UsetheSSHprotocolwithBitbucketCloud-KnownhostorBitbucket%27spublickeyfingerprints
[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source
diff --git a/doc/integration/img/bitbucket_oauth_settings_page.png b/doc/integration/img/bitbucket_oauth_settings_page.png
index 8dbee9762d7..21ce82a6074 100644
--- a/doc/integration/img/bitbucket_oauth_settings_page.png
+++ b/doc/integration/img/bitbucket_oauth_settings_page.png
Binary files differ
diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png
deleted file mode 100644
index df55a081803..00000000000
--- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_grant_access.png
+++ /dev/null
Binary files differ
diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png
deleted file mode 100644
index 5253889d251..00000000000
--- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_new_project.png
+++ /dev/null
Binary files differ
diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png
deleted file mode 100644
index ffa87ce5b2e..00000000000
--- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_bitbucket.png
+++ /dev/null
Binary files differ
diff --git a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png b/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png
deleted file mode 100644
index 1a5661de75d..00000000000
--- a/doc/workflow/importing/bitbucket_importer/bitbucket_import_select_project.png
+++ /dev/null
Binary files differ
diff --git a/doc/workflow/importing/img/bitbucket_import_grant_access.png b/doc/workflow/importing/img/bitbucket_import_grant_access.png
new file mode 100644
index 00000000000..429904e621d
--- /dev/null
+++ b/doc/workflow/importing/img/bitbucket_import_grant_access.png
Binary files differ
diff --git a/doc/workflow/importing/img/bitbucket_import_new_project.png b/doc/workflow/importing/img/bitbucket_import_new_project.png
new file mode 100644
index 00000000000..8ed528c2f09
--- /dev/null
+++ b/doc/workflow/importing/img/bitbucket_import_new_project.png
Binary files differ
diff --git a/doc/workflow/importing/img/bitbucket_import_select_project.png b/doc/workflow/importing/img/bitbucket_import_select_project.png
new file mode 100644
index 00000000000..1bca6166ec8
--- /dev/null
+++ b/doc/workflow/importing/img/bitbucket_import_select_project.png
Binary files differ
diff --git a/doc/workflow/importing/img/import_projects_from_github_new_project_page.png b/doc/workflow/importing/img/import_projects_from_github_new_project_page.png
deleted file mode 100644
index b23ade4480c..00000000000
--- a/doc/workflow/importing/img/import_projects_from_github_new_project_page.png
+++ /dev/null
Binary files differ
diff --git a/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png b/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png
index f50d9266991..1ccb38a815e 100644
--- a/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png
+++ b/doc/workflow/importing/img/import_projects_from_github_select_auth_method.png
Binary files differ
diff --git a/doc/workflow/importing/img/import_projects_from_new_project_page.png b/doc/workflow/importing/img/import_projects_from_new_project_page.png
new file mode 100644
index 00000000000..97ca30b2087
--- /dev/null
+++ b/doc/workflow/importing/img/import_projects_from_new_project_page.png
Binary files differ
diff --git a/doc/workflow/importing/import_projects_from_bitbucket.md b/doc/workflow/importing/import_projects_from_bitbucket.md
index 520c4216295..b6d47e5afa2 100644
--- a/doc/workflow/importing/import_projects_from_bitbucket.md
+++ b/doc/workflow/importing/import_projects_from_bitbucket.md
@@ -1,26 +1,61 @@
# Import your project from Bitbucket to GitLab
-It takes just a few steps to import your existing Bitbucket projects to GitLab. But keep in mind that it is possible only if Bitbucket support is enabled on your GitLab instance. You can read more about Bitbucket support [here](../../integration/bitbucket.md).
+Import your projects from Bitbucket to GitLab with minimal effort.
-* Sign in to GitLab.com and go to your dashboard
+## Overview
-* Click on "New project"
+>**Note:**
+The [Bitbucket integration][bb-import] must be first enabled in order to be
+able to import your projects from Bitbucket. Ask your GitLab administrator
+to enable this if not already.
-![New project in GitLab](bitbucket_importer/bitbucket_import_new_project.png)
+- At its current state, the Bitbucket importer can import:
+ - the repository description (GitLab 7.7+)
+ - the Git repository data (GitLab 7.7+)
+ - the issues (GitLab 7.7+)
+ - the issue comments (GitLab 8.15+)
+ - the pull requests (GitLab 8.4+)
+ - the pull request comments (GitLab 8.15+)
+ - the milestones (GitLab 8.15+)
+- References to pull requests and issues are preserved (GitLab 8.7+)
+- Repository public access is retained. If a repository is private in Bitbucket
+ it will be created as private in GitLab as well.
-* Click on the "Bitbucket" button
-![Bitbucket](bitbucket_importer/bitbucket_import_select_bitbucket.png)
+## How it works
-* Grant GitLab access to your Bitbucket account
+When issues/pull requests are being imported, the Bitbucket importer tries to find
+the Bitbucket author/assignee in GitLab's database using the Bitbucket ID. For this
+to work, the Bitbucket author/assignee should have signed in beforehand in GitLab
+and [**associated their Bitbucket account**][social sign-in]. If the user is not
+found in GitLab's database, the project creator (most of the times the current
+user that started the import process) is set as the author, but a reference on
+the issue about the original Bitbucket author is kept.
-![Grant access](bitbucket_importer/bitbucket_import_grant_access.png)
+The importer will create any new namespaces (groups) if they don't exist or in
+the case the namespace is taken, the repository will be imported under the user's
+namespace that started the import process.
-* Click on the projects that you'd like to import or "Import all projects"
+## Importing your Bitbucket repositories
-![Import projects](bitbucket_importer/bitbucket_import_select_project.png)
+1. Sign in to GitLab and go to your dashboard.
+1. Click on **New project**.
-A new GitLab project will be created with your imported data.
+ ![New project in GitLab](img/bitbucket_import_new_project.png)
-### Note
-Milestones and wiki pages are not imported from Bitbucket.
+1. Click on the "Bitbucket" button
+
+ ![Bitbucket](img/import_projects_from_new_project_page.png)
+
+1. Grant GitLab access to your Bitbucket account
+
+ ![Grant access](img/bitbucket_import_grant_access.png)
+
+1. Click on the projects that you'd like to import or **Import all projects**.
+ You can also select the namespace under which each project will be
+ imported.
+
+ ![Import projects](img/bitbucket_import_select_project.png)
+
+[bb-import]: ../../integration/bitbucket.md
+[social sign-in]: ../../user/profile/account/social_sign_in.md
diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md
index c36dfdb78ec..b3660aa8030 100644
--- a/doc/workflow/importing/import_projects_from_github.md
+++ b/doc/workflow/importing/import_projects_from_github.md
@@ -40,7 +40,7 @@ namespace that started the import process.
The importer page is visible when you create a new project.
-![New project page on GitLab](img/import_projects_from_github_new_project_page.png)
+![New project page on GitLab](img/import_projects_from_new_project_page.png)
Click on the **GitHub** link and the import authorization process will start.
There are two ways to authorize access to your GitHub repositories:
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb
new file mode 100644
index 00000000000..f8ee7e0f9ae
--- /dev/null
+++ b/lib/bitbucket/client.rb
@@ -0,0 +1,58 @@
+module Bitbucket
+ class Client
+ attr_reader :connection
+
+ def initialize(options = {})
+ @connection = Connection.new(options)
+ end
+
+ def issues(repo)
+ path = "/repositories/#{repo}/issues"
+ get_collection(path, :issue)
+ end
+
+ def issue_comments(repo, issue_id)
+ path = "/repositories/#{repo}/issues/#{issue_id}/comments"
+ get_collection(path, :comment)
+ end
+
+ def pull_requests(repo)
+ path = "/repositories/#{repo}/pullrequests?state=ALL"
+ get_collection(path, :pull_request)
+ end
+
+ def pull_request_comments(repo, pull_request)
+ path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments"
+ get_collection(path, :pull_request_comment)
+ end
+
+ def pull_request_diff(repo, pull_request)
+ path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff"
+ connection.get(path)
+ end
+
+ def repo(name)
+ parsed_response = connection.get("/repositories/#{name}")
+ Representation::Repo.new(parsed_response)
+ end
+
+ def repos
+ path = "/repositories?role=member"
+ get_collection(path, :repo)
+ end
+
+ def user
+ @user ||= begin
+ parsed_response = connection.get('/user')
+ Representation::User.new(parsed_response)
+ end
+ end
+
+ private
+
+ def get_collection(path, type)
+ paginator = Paginator.new(connection, path, type)
+ Collection.new(paginator)
+ end
+ end
+end
diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb
new file mode 100644
index 00000000000..3a9379ff680
--- /dev/null
+++ b/lib/bitbucket/collection.rb
@@ -0,0 +1,21 @@
+module Bitbucket
+ class Collection < Enumerator
+ def initialize(paginator)
+ super() do |yielder|
+ loop do
+ paginator.items.each { |item| yielder << item }
+ end
+ end
+
+ lazy
+ end
+
+ def method_missing(method, *args)
+ return super unless self.respond_to?(method)
+
+ self.send(method, *args) do |item|
+ block_given? ? yield(item) : item
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb
new file mode 100644
index 00000000000..7e55cf4deab
--- /dev/null
+++ b/lib/bitbucket/connection.rb
@@ -0,0 +1,69 @@
+module Bitbucket
+ class Connection
+ DEFAULT_API_VERSION = '2.0'
+ DEFAULT_BASE_URI = 'https://api.bitbucket.org/'
+ DEFAULT_QUERY = {}
+
+ attr_reader :expires_at, :expires_in, :refresh_token, :token
+
+ def initialize(options = {})
+ @api_version = options.fetch(:api_version, DEFAULT_API_VERSION)
+ @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI)
+ @default_query = options.fetch(:query, DEFAULT_QUERY)
+
+ @token = options[:token]
+ @expires_at = options[:expires_at]
+ @expires_in = options[:expires_in]
+ @refresh_token = options[:refresh_token]
+ end
+
+ def get(path, extra_query = {})
+ refresh! if expired?
+
+ response = connection.get(build_url(path), params: @default_query.merge(extra_query))
+ response.parsed
+ end
+
+ def expired?
+ connection.expired?
+ end
+
+ def refresh!
+ response = connection.refresh!
+
+ @token = response.token
+ @expires_at = response.expires_at
+ @expires_in = response.expires_in
+ @refresh_token = response.refresh_token
+ @connection = nil
+ end
+
+ private
+
+ def client
+ @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+ end
+
+ def connection
+ @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in)
+ end
+
+ def build_url(path)
+ return path if path.starts_with?(root_url)
+
+ "#{root_url}#{path}"
+ end
+
+ def root_url
+ @root_url ||= "#{@base_uri}#{@api_version}"
+ end
+
+ def provider
+ Gitlab::OAuth::Provider.config_for('bitbucket')
+ end
+
+ def options
+ OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
+ end
+ end
+end
diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb
new file mode 100644
index 00000000000..5e2eb57bb0e
--- /dev/null
+++ b/lib/bitbucket/error/unauthorized.rb
@@ -0,0 +1,6 @@
+module Bitbucket
+ module Error
+ class Unauthorized < StandardError
+ end
+ end
+end
diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb
new file mode 100644
index 00000000000..2b0a3fe7b1a
--- /dev/null
+++ b/lib/bitbucket/page.rb
@@ -0,0 +1,34 @@
+module Bitbucket
+ class Page
+ attr_reader :attrs, :items
+
+ def initialize(raw, type)
+ @attrs = parse_attrs(raw)
+ @items = parse_values(raw, representation_class(type))
+ end
+
+ def next?
+ attrs.fetch(:next, false)
+ end
+
+ def next
+ attrs.fetch(:next)
+ end
+
+ private
+
+ def parse_attrs(raw)
+ raw.slice(*%w(size page pagelen next previous)).symbolize_keys
+ end
+
+ def parse_values(raw, bitbucket_rep_class)
+ return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+ bitbucket_rep_class.decorate(raw['values'])
+ end
+
+ def representation_class(type)
+ Bitbucket::Representation.const_get(type.to_s.camelize)
+ end
+ end
+end
diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb
new file mode 100644
index 00000000000..135d0d55674
--- /dev/null
+++ b/lib/bitbucket/paginator.rb
@@ -0,0 +1,36 @@
+module Bitbucket
+ class Paginator
+ PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100.
+
+ def initialize(connection, url, type)
+ @connection = connection
+ @type = type
+ @url = url
+ @page = nil
+ end
+
+ def items
+ raise StopIteration unless has_next_page?
+
+ @page = fetch_next_page
+ @page.items
+ end
+
+ private
+
+ attr_reader :connection, :page, :url, :type
+
+ def has_next_page?
+ page.nil? || page.next?
+ end
+
+ def next_url
+ page.nil? ? url : page.next
+ end
+
+ def fetch_next_page
+ parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
+ Page.new(parsed_response, type)
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb
new file mode 100644
index 00000000000..94adaacc9b5
--- /dev/null
+++ b/lib/bitbucket/representation/base.rb
@@ -0,0 +1,17 @@
+module Bitbucket
+ module Representation
+ class Base
+ def initialize(raw)
+ @raw = raw
+ end
+
+ def self.decorate(entries)
+ entries.map { |entry| new(entry)}
+ end
+
+ private
+
+ attr_reader :raw
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb
new file mode 100644
index 00000000000..4937aa9728f
--- /dev/null
+++ b/lib/bitbucket/representation/comment.rb
@@ -0,0 +1,27 @@
+module Bitbucket
+ module Representation
+ class Comment < Representation::Base
+ def author
+ user['username']
+ end
+
+ def note
+ raw.fetch('content', {}).fetch('raw', nil)
+ end
+
+ def created_at
+ raw['created_on']
+ end
+
+ def updated_at
+ raw['updated_on'] || raw['created_on']
+ end
+
+ private
+
+ def user
+ raw.fetch('user', {})
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb
new file mode 100644
index 00000000000..054064395c3
--- /dev/null
+++ b/lib/bitbucket/representation/issue.rb
@@ -0,0 +1,53 @@
+module Bitbucket
+ module Representation
+ class Issue < Representation::Base
+ CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze
+
+ def iid
+ raw['id']
+ end
+
+ def kind
+ raw['kind']
+ end
+
+ def author
+ raw.fetch('reporter', {}).fetch('username', nil)
+ end
+
+ def description
+ raw.fetch('content', {}).fetch('raw', nil)
+ end
+
+ def state
+ closed? ? 'closed' : 'opened'
+ end
+
+ def title
+ raw['title']
+ end
+
+ def milestone
+ raw['milestone']['name'] if raw['milestone'].present?
+ end
+
+ def created_at
+ raw['created_on']
+ end
+
+ def updated_at
+ raw['edited_on']
+ end
+
+ def to_s
+ iid
+ end
+
+ private
+
+ def closed?
+ CLOSED_STATUS.include?(raw['state'])
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb
new file mode 100644
index 00000000000..eebf8093380
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request.rb
@@ -0,0 +1,65 @@
+module Bitbucket
+ module Representation
+ class PullRequest < Representation::Base
+ def author
+ raw.fetch('author', {}).fetch('username', nil)
+ end
+
+ def description
+ raw['description']
+ end
+
+ def iid
+ raw['id']
+ end
+
+ def state
+ if raw['state'] == 'MERGED'
+ 'merged'
+ elsif raw['state'] == 'DECLINED'
+ 'closed'
+ else
+ 'opened'
+ end
+ end
+
+ def created_at
+ raw['created_on']
+ end
+
+ def updated_at
+ raw['updated_on']
+ end
+
+ def title
+ raw['title']
+ end
+
+ def source_branch_name
+ source_branch.fetch('branch', {}).fetch('name', nil)
+ end
+
+ def source_branch_sha
+ source_branch.fetch('commit', {}).fetch('hash', nil)
+ end
+
+ def target_branch_name
+ target_branch.fetch('branch', {}).fetch('name', nil)
+ end
+
+ def target_branch_sha
+ target_branch.fetch('commit', {}).fetch('hash', nil)
+ end
+
+ private
+
+ def source_branch
+ raw['source']
+ end
+
+ def target_branch
+ raw['destination']
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb
new file mode 100644
index 00000000000..4f8efe03bae
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request_comment.rb
@@ -0,0 +1,39 @@
+module Bitbucket
+ module Representation
+ class PullRequestComment < Comment
+ def iid
+ raw['id']
+ end
+
+ def file_path
+ inline.fetch('path')
+ end
+
+ def old_pos
+ inline.fetch('from')
+ end
+
+ def new_pos
+ inline.fetch('to')
+ end
+
+ def parent_id
+ raw.fetch('parent', {}).fetch('id', nil)
+ end
+
+ def inline?
+ raw.has_key?('inline')
+ end
+
+ def has_parent?
+ raw.has_key?('parent')
+ end
+
+ private
+
+ def inline
+ raw.fetch('inline', {})
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb
new file mode 100644
index 00000000000..8969ecd1c19
--- /dev/null
+++ b/lib/bitbucket/representation/repo.rb
@@ -0,0 +1,67 @@
+module Bitbucket
+ module Representation
+ class Repo < Representation::Base
+ attr_reader :owner, :slug
+
+ def initialize(raw)
+ super(raw)
+ end
+
+ def owner_and_slug
+ @owner_and_slug ||= full_name.split('/', 2)
+ end
+
+ def owner
+ owner_and_slug.first
+ end
+
+ def slug
+ owner_and_slug.last
+ end
+
+ def clone_url(token = nil)
+ url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href')
+
+ if token.present?
+ clone_url = URI::parse(url)
+ clone_url.user = "x-token-auth:#{token}"
+ clone_url.to_s
+ else
+ url
+ end
+ end
+
+ def description
+ raw['description']
+ end
+
+ def full_name
+ raw['full_name']
+ end
+
+ def issues_enabled?
+ raw['has_issues']
+ end
+
+ def name
+ raw['name']
+ end
+
+ def valid?
+ raw['scm'] == 'git'
+ end
+
+ def visibility_level
+ if raw['is_private']
+ Gitlab::VisibilityLevel::PRIVATE
+ else
+ Gitlab::VisibilityLevel::PUBLIC
+ end
+ end
+
+ def to_s
+ full_name
+ end
+ end
+ end
+end
diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb
new file mode 100644
index 00000000000..ba6b7667b49
--- /dev/null
+++ b/lib/bitbucket/representation/user.rb
@@ -0,0 +1,9 @@
+module Bitbucket
+ module Representation
+ class User < Representation::Base
+ def username
+ raw['username']
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb
deleted file mode 100644
index 7298152e7e9..00000000000
--- a/lib/gitlab/bitbucket_import.rb
+++ /dev/null
@@ -1,6 +0,0 @@
-module Gitlab
- module BitbucketImport
- mattr_accessor :public_key
- @public_key = nil
- end
-end
diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
deleted file mode 100644
index 8d1ad62fae0..00000000000
--- a/lib/gitlab/bitbucket_import/client.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-module Gitlab
- module BitbucketImport
- class Client
- class Unauthorized < StandardError; end
-
- attr_reader :consumer, :api
-
- def self.from_project(project)
- import_data_credentials = project.import_data.credentials if project.import_data
- if import_data_credentials && import_data_credentials[:bb_session]
- token = import_data_credentials[:bb_session][:bitbucket_access_token]
- token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret]
- new(token, token_secret)
- else
- raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}"
- end
- end
-
- def initialize(access_token = nil, access_token_secret = nil)
- @consumer = ::OAuth::Consumer.new(
- config.app_id,
- config.app_secret,
- bitbucket_options
- )
-
- if access_token && access_token_secret
- @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
- end
- end
-
- def request_token(redirect_uri)
- request_token = consumer.get_request_token(oauth_callback: redirect_uri)
-
- {
- oauth_token: request_token.token,
- oauth_token_secret: request_token.secret,
- oauth_callback_confirmed: request_token.callback_confirmed?.to_s
- }
- end
-
- def authorize_url(request_token, redirect_uri)
- request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
- if request_token.callback_confirmed?
- request_token.authorize_url
- else
- request_token.authorize_url(oauth_callback: redirect_uri)
- end
- end
-
- def get_token(request_token, oauth_verifier, redirect_uri)
- request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
- if request_token.callback_confirmed?
- request_token.get_access_token(oauth_verifier: oauth_verifier)
- else
- request_token.get_access_token(oauth_callback: redirect_uri)
- end
- end
-
- def user
- JSON.parse(get("/api/1.0/user").body)
- end
-
- def issues(project_identifier)
- all_issues = []
- offset = 0
- per_page = 50 # Maximum number allowed by Bitbucket
- index = 0
-
- begin
- issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
- # Find out how many total issues are present
- total = issues["count"] if index == 0
- all_issues.concat(issues["issues"])
- offset += issues["issues"].count
- index += 1
- end while all_issues.count < total
-
- all_issues
- end
-
- def issue_comments(project_identifier, issue_id)
- comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
- comments.sort_by { |comment| comment["utc_created_on"] }
- end
-
- def project(project_identifier)
- JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body)
- end
-
- def find_deploy_key(project_identifier, key)
- JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
- deploy_key["key"].chomp == key.chomp
- end
- end
-
- def add_deploy_key(project_identifier, key)
- deploy_key = find_deploy_key(project_identifier, key)
- return if deploy_key
-
- JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
- end
-
- def delete_deploy_key(project_identifier, key)
- deploy_key = find_deploy_key(project_identifier, key)
- return unless deploy_key
-
- api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204"
- end
-
- def projects
- JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
- end
-
- def incompatible_projects
- JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" }
- end
-
- private
-
- def get(url)
- response = api.get(url)
- raise Unauthorized if (400..499).cover?(response.code.to_i)
-
- response
- end
-
- def issue_api_endpoint(project_identifier, per_page, offset)
- "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
- end
-
- def config
- Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" }
- end
-
- def bitbucket_options
- OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys
- end
- end
- end
-end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index f4b5097adb1..7d2f92d577a 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -1,84 +1,234 @@
module Gitlab
module BitbucketImport
class Importer
- attr_reader :project, :client
+ LABELS = [{ title: 'bug', color: '#FF0000' },
+ { title: 'enhancement', color: '#428BCA' },
+ { title: 'proposal', color: '#69D100' },
+ { title: 'task', color: '#7F8C8D' }].freeze
+
+ attr_reader :project, :client, :errors, :users
def initialize(project)
@project = project
- @client = Client.from_project(@project)
+ @client = Bitbucket::Client.new(project.import_data.credentials)
@formatter = Gitlab::ImportFormatter.new
+ @labels = {}
+ @errors = []
+ @users = {}
end
def execute
- import_issues if has_issues?
+ import_issues
+ import_pull_requests
+ handle_errors
true
- rescue ActiveRecord::RecordInvalid => e
- raise Projects::ImportService::Error.new, e.message
- ensure
- Gitlab::BitbucketImport::KeyDeleter.new(project).execute
end
private
- def gitlab_user_id(project, bitbucket_id)
- if bitbucket_id
- user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
- (user && user.id) || project.creator_id
- else
- project.creator_id
- end
+ def handle_errors
+ return unless errors.any?
+
+ project.update_column(:import_error, {
+ message: 'The remote data could not be fully imported.',
+ errors: errors
+ }.to_json)
end
- def identifier
- project.import_source
+ def gitlab_user_id(project, username)
+ find_user_id(username) || project.creator_id
end
- def has_issues?
- client.project(identifier)["has_issues"]
+ def find_user_id(username)
+ return nil unless username
+
+ return users[username] if users.key?(username)
+
+ users[username] = User.select(:id)
+ .joins(:identities)
+ .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username)
+ .try(:id)
+ end
+
+ def repo
+ @repo ||= client.repo(project.import_source)
end
def import_issues
- issues = client.issues(identifier)
+ return unless repo.issues_enabled?
+
+ create_labels
+
+ client.issues(repo).each do |issue|
+ begin
+ description = ''
+ description += @formatter.author_line(issue.author) unless find_user_id(issue.author)
+ description += issue.description
+
+ label_name = issue.kind
+ milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil
+
+ gitlab_issue = project.issues.create!(
+ iid: issue.iid,
+ title: issue.title,
+ description: description,
+ state: issue.state,
+ author_id: gitlab_user_id(project, issue.author),
+ milestone: milestone,
+ created_at: issue.created_at,
+ updated_at: issue.updated_at
+ )
+
+ gitlab_issue.labels << @labels[label_name]
+
+ import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted?
+ rescue StandardError => e
+ errors << { type: :issue, iid: issue.iid, errors: e.message }
+ end
+ end
+ end
+
+ def import_issue_comments(issue, gitlab_issue)
+ client.issue_comments(repo, issue.iid).each do |comment|
+ # The note can be blank for issue service messages like "Changed title: ..."
+ # We would like to import those comments as well but there is no any
+ # specific parameter that would allow to process them, it's just an empty comment.
+ # To prevent our importer from just crashing or from creating useless empty comments
+ # we do this check.
+ next unless comment.note.present?
+
+ note = ''
+ note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
+ note += comment.note
+
+ begin
+ gitlab_issue.notes.create!(
+ project: project,
+ note: note,
+ author_id: gitlab_user_id(project, comment.author),
+ created_at: comment.created_at,
+ updated_at: comment.updated_at
+ )
+ rescue StandardError => e
+ errors << { type: :issue_comment, iid: issue.iid, errors: e.message }
+ end
+ end
+ end
- issues.each do |issue|
- body = ''
- reporter = nil
- author = 'Anonymous'
+ def create_labels
+ LABELS.each do |label|
+ @labels[label[:title]] = project.labels.create!(label)
+ end
+ end
- if issue["reported_by"] && issue["reported_by"]["username"]
- reporter = issue["reported_by"]["username"]
- author = reporter
+ def import_pull_requests
+ pull_requests = client.pull_requests(repo)
+
+ pull_requests.each do |pull_request|
+ begin
+ description = ''
+ description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author)
+ description += pull_request.description
+
+ merge_request = project.merge_requests.create(
+ iid: pull_request.iid,
+ title: pull_request.title,
+ description: description,
+ source_project: project,
+ source_branch: pull_request.source_branch_name,
+ source_branch_sha: pull_request.source_branch_sha,
+ target_project: project,
+ target_branch: pull_request.target_branch_name,
+ target_branch_sha: pull_request.target_branch_sha,
+ state: pull_request.state,
+ author_id: gitlab_user_id(project, pull_request.author),
+ assignee_id: nil,
+ created_at: pull_request.created_at,
+ updated_at: pull_request.updated_at
+ )
+
+ import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
+ rescue StandardError => e
+ errors << { type: :pull_request, iid: pull_request.iid, errors: e.message }
end
+ end
+ end
- body = @formatter.author_line(author)
- body += issue["content"]
+ def import_pull_request_comments(pull_request, merge_request)
+ comments = client.pull_request_comments(repo, pull_request.iid)
- comments = client.issue_comments(identifier, issue["local_id"])
+ inline_comments, pr_comments = comments.partition(&:inline?)
- if comments.any?
- body += @formatter.comments_header
- end
+ import_inline_comments(inline_comments, pull_request, merge_request)
+ import_standalone_pr_comments(pr_comments, merge_request)
+ end
- comments.each do |comment|
- author = 'Anonymous'
+ def import_inline_comments(inline_comments, pull_request, merge_request)
+ line_code_map = {}
- if comment["author_info"] && comment["author_info"]["username"]
- author = comment["author_info"]["username"]
- end
+ children, parents = inline_comments.partition(&:has_parent?)
+
+ # The Bitbucket API returns threaded replies as parent-child
+ # relationships. We assume that the child can appear in any order in
+ # the JSON.
+ parents.each do |comment|
+ line_code_map[comment.iid] = generate_line_code(comment)
+ end
- body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
+ children.each do |comment|
+ line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil)
+ end
+
+ inline_comments.each do |comment|
+ begin
+ attributes = pull_request_comment_attributes(comment)
+ attributes.merge!(
+ position: build_position(merge_request, comment),
+ line_code: line_code_map.fetch(comment.iid),
+ type: 'DiffNote')
+
+ merge_request.notes.create!(attributes)
+ rescue StandardError => e
+ errors << { type: :pull_request, iid: comment.iid, errors: e.message }
end
+ end
+ end
- project.issues.create!(
- description: body,
- title: issue["title"],
- state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened',
- author_id: gitlab_user_id(project, reporter)
- )
+ def build_position(merge_request, pr_comment)
+ params = {
+ diff_refs: merge_request.diff_refs,
+ old_path: pr_comment.file_path,
+ new_path: pr_comment.file_path,
+ old_line: pr_comment.old_pos,
+ new_line: pr_comment.new_pos
+ }
+
+ Gitlab::Diff::Position.new(params)
+ end
+
+ def import_standalone_pr_comments(pr_comments, merge_request)
+ pr_comments.each do |comment|
+ begin
+ merge_request.notes.create!(pull_request_comment_attributes(comment))
+ rescue StandardError => e
+ errors << { type: :pull_request, iid: comment.iid, errors: e.message }
+ end
end
- rescue ActiveRecord::RecordInvalid => e
- raise Projects::ImportService::Error, e.message
+ end
+
+ def generate_line_code(pr_comment)
+ Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos)
+ end
+
+ def pull_request_comment_attributes(comment)
+ {
+ project: project,
+ note: comment.note,
+ author_id: gitlab_user_id(project, comment.author),
+ created_at: comment.created_at,
+ updated_at: comment.updated_at
+ }
end
end
end
diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb
deleted file mode 100644
index 0b63f025d0a..00000000000
--- a/lib/gitlab/bitbucket_import/key_adder.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-module Gitlab
- module BitbucketImport
- class KeyAdder
- attr_reader :repo, :current_user, :client
-
- def initialize(repo, current_user, access_params)
- @repo, @current_user = repo, current_user
- @client = Client.new(access_params[:bitbucket_access_token],
- access_params[:bitbucket_access_token_secret])
- end
-
- def execute
- return false unless BitbucketImport.public_key.present?
-
- project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
- client.add_deploy_key(project_identifier, BitbucketImport.public_key)
-
- true
- rescue
- false
- end
- end
- end
-end
diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb
deleted file mode 100644
index e03c3155b3e..00000000000
--- a/lib/gitlab/bitbucket_import/key_deleter.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-module Gitlab
- module BitbucketImport
- class KeyDeleter
- attr_reader :project, :current_user, :client
-
- def initialize(project)
- @project = project
- @current_user = project.creator
- @client = Client.from_project(@project)
- end
-
- def execute
- return false unless BitbucketImport.public_key.present?
-
- client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
-
- true
- rescue
- false
- end
- end
- end
-end
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index b90ef0b0fba..eb03882ab26 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,10 +1,11 @@
module Gitlab
module BitbucketImport
class ProjectCreator
- attr_reader :repo, :namespace, :current_user, :session_data
+ attr_reader :repo, :name, :namespace, :current_user, :session_data
- def initialize(repo, namespace, current_user, session_data)
+ def initialize(repo, name, namespace, current_user, session_data)
@repo = repo
+ @name = name
@namespace = namespace
@current_user = current_user
@session_data = session_data
@@ -13,15 +14,15 @@ module Gitlab
def execute
::Projects::CreateService.new(
current_user,
- name: repo["name"],
- path: repo["slug"],
- description: repo["description"],
+ name: name,
+ path: name,
+ description: repo.description,
namespace_id: namespace.id,
- visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
- import_type: "bitbucket",
- import_source: "#{repo["owner"]}/#{repo["slug"]}",
- import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
- import_data: { credentials: { bb_session: session_data } }
+ visibility_level: repo.visibility_level,
+ import_type: 'bitbucket',
+ import_source: repo.full_name,
+ import_url: repo.clone_url(session_data[:token]),
+ import_data: { credentials: session_data }
).execute
end
end
diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb
new file mode 100644
index 00000000000..5a7d67c2390
--- /dev/null
+++ b/lib/omniauth/strategies/bitbucket.rb
@@ -0,0 +1,41 @@
+require 'omniauth-oauth2'
+
+module OmniAuth
+ module Strategies
+ class Bitbucket < OmniAuth::Strategies::OAuth2
+ option :name, 'bitbucket'
+
+ option :client_options, {
+ site: 'https://bitbucket.org',
+ authorize_url: 'https://bitbucket.org/site/oauth2/authorize',
+ token_url: 'https://bitbucket.org/site/oauth2/access_token'
+ }
+
+ uid do
+ raw_info['username']
+ end
+
+ info do
+ {
+ name: raw_info['display_name'],
+ avatar: raw_info['links']['avatar']['href'],
+ email: primary_email
+ }
+ end
+
+ def raw_info
+ @raw_info ||= access_token.get('api/2.0/user').parsed
+ end
+
+ def primary_email
+ primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] }
+ primary && primary['email'] || nil
+ end
+
+ def emails
+ email_response = access_token.get('api/2.0/user/emails').parsed
+ @emails ||= email_response && email_response['values'] || nil
+ end
+ end
+ end
+end
diff --git a/spec/controllers/import/bitbucket_controller_spec.rb b/spec/controllers/import/bitbucket_controller_spec.rb
index 1d3c9fbbe2f..ce7c0b334ee 100644
--- a/spec/controllers/import/bitbucket_controller_spec.rb
+++ b/spec/controllers/import/bitbucket_controller_spec.rb
@@ -6,11 +6,11 @@ describe Import::BitbucketController do
let(:user) { create(:user) }
let(:token) { "asdasd12345" }
let(:secret) { "sekrettt" }
- let(:access_params) { { bitbucket_access_token: token, bitbucket_access_token_secret: secret } }
+ let(:refresh_token) { SecureRandom.hex(15) }
+ let(:access_params) { { token: token, expires_at: nil, expires_in: nil, refresh_token: nil } }
def assign_session_tokens
- session[:bitbucket_access_token] = token
- session[:bitbucket_access_token_secret] = secret
+ session[:bitbucket_token] = token
end
before do
@@ -24,29 +24,36 @@ describe Import::BitbucketController do
end
it "updates access token" do
- access_token = double(token: token, secret: secret)
- allow_any_instance_of(Gitlab::BitbucketImport::Client).
+ expires_at = Time.now + 1.day
+ expires_in = 1.day
+ access_token = double(token: token,
+ secret: secret,
+ expires_at: expires_at,
+ expires_in: expires_in,
+ refresh_token: refresh_token)
+ allow_any_instance_of(OAuth2::Client).
to receive(:get_token).and_return(access_token)
stub_omniauth_provider('bitbucket')
get :callback
- expect(session[:bitbucket_access_token]).to eq(token)
- expect(session[:bitbucket_access_token_secret]).to eq(secret)
+ expect(session[:bitbucket_token]).to eq(token)
+ expect(session[:bitbucket_refresh_token]).to eq(refresh_token)
+ expect(session[:bitbucket_expires_at]).to eq(expires_at)
+ expect(session[:bitbucket_expires_in]).to eq(expires_in)
expect(controller).to redirect_to(status_import_bitbucket_url)
end
end
describe "GET status" do
before do
- @repo = OpenStruct.new(slug: 'vim', owner: 'asd')
+ @repo = double(slug: 'vim', owner: 'asd', full_name: 'asd/vim', "valid?" => true)
assign_session_tokens
end
it "assigns variables" do
@project = create(:project, import_type: 'bitbucket', creator_id: user.id)
- client = stub_client(projects: [@repo])
- allow(client).to receive(:incompatible_projects).and_return([])
+ allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo])
get :status
@@ -57,7 +64,7 @@ describe Import::BitbucketController do
it "does not show already added project" do
@project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim')
- stub_client(projects: [@repo])
+ allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo])
get :status
@@ -70,19 +77,16 @@ describe Import::BitbucketController do
let(:bitbucket_username) { user.username }
let(:bitbucket_user) do
- { user: { username: bitbucket_username } }.with_indifferent_access
+ double(username: bitbucket_username)
end
let(:bitbucket_repo) do
- { slug: "vim", owner: bitbucket_username }.with_indifferent_access
+ double(slug: "vim", owner: bitbucket_username, name: 'vim')
end
before do
- allow(Gitlab::BitbucketImport::KeyAdder).
- to receive(:new).with(bitbucket_repo, user, access_params).
- and_return(double(execute: true))
-
- stub_client(user: bitbucket_user, project: bitbucket_repo)
+ allow_any_instance_of(Bitbucket::Client).to receive(:repo).and_return(bitbucket_repo)
+ allow_any_instance_of(Bitbucket::Client).to receive(:user).and_return(bitbucket_user)
assign_session_tokens
end
@@ -90,7 +94,7 @@ describe Import::BitbucketController do
context "when the Bitbucket user and GitLab user's usernames match" do
it "takes the current user's namespace" do
expect(Gitlab::BitbucketImport::ProjectCreator).
- to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+ to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
and_return(double(execute: true))
post :create, format: :js
@@ -102,7 +106,7 @@ describe Import::BitbucketController do
it "takes the current user's namespace" do
expect(Gitlab::BitbucketImport::ProjectCreator).
- to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+ to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
and_return(double(execute: true))
post :create, format: :js
@@ -114,7 +118,7 @@ describe Import::BitbucketController do
let(:other_username) { "someone_else" }
before do
- bitbucket_repo["owner"] = other_username
+ allow(bitbucket_repo).to receive(:owner).and_return(other_username)
end
context "when a namespace with the Bitbucket user's username already exists" do
@@ -123,7 +127,7 @@ describe Import::BitbucketController do
context "when the namespace is owned by the GitLab user" do
it "takes the existing namespace" do
expect(Gitlab::BitbucketImport::ProjectCreator).
- to receive(:new).with(bitbucket_repo, existing_namespace, user, access_params).
+ to receive(:new).with(bitbucket_repo, bitbucket_repo.name, existing_namespace, user, access_params).
and_return(double(execute: true))
post :create, format: :js
@@ -156,7 +160,7 @@ describe Import::BitbucketController do
it "takes the new namespace" do
expect(Gitlab::BitbucketImport::ProjectCreator).
- to receive(:new).with(bitbucket_repo, an_instance_of(Group), user, access_params).
+ to receive(:new).with(bitbucket_repo, bitbucket_repo.name, an_instance_of(Group), user, access_params).
and_return(double(execute: true))
post :create, format: :js
@@ -177,7 +181,7 @@ describe Import::BitbucketController do
it "takes the current user's namespace" do
expect(Gitlab::BitbucketImport::ProjectCreator).
- to receive(:new).with(bitbucket_repo, user.namespace, user, access_params).
+ to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params).
and_return(double(execute: true))
post :create, format: :js
diff --git a/spec/lib/bitbucket/collection_spec.rb b/spec/lib/bitbucket/collection_spec.rb
new file mode 100644
index 00000000000..015a7f80e03
--- /dev/null
+++ b/spec/lib/bitbucket/collection_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+# Emulates paginator. It returns 2 pages with results
+class TestPaginator
+ def initialize
+ @current_page = 0
+ end
+
+ def items
+ @current_page += 1
+
+ raise StopIteration if @current_page > 2
+
+ ["result_1_page_#{@current_page}", "result_2_page_#{@current_page}"]
+ end
+end
+
+describe Bitbucket::Collection do
+ it "iterates paginator" do
+ collection = described_class.new(TestPaginator.new)
+
+ expect(collection.to_a).to match(["result_1_page_1", "result_2_page_1", "result_1_page_2", "result_2_page_2"])
+ end
+end
diff --git a/spec/lib/bitbucket/connection_spec.rb b/spec/lib/bitbucket/connection_spec.rb
new file mode 100644
index 00000000000..14faeb231a9
--- /dev/null
+++ b/spec/lib/bitbucket/connection_spec.rb
@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Bitbucket::Connection do
+ before do
+ allow_any_instance_of(described_class).to receive(:provider).and_return(double(app_id: '', app_secret: ''))
+ end
+
+ describe '#get' do
+ it 'calls OAuth2::AccessToken::get' do
+ expect_any_instance_of(OAuth2::AccessToken).to receive(:get).and_return(double(parsed: true))
+
+ connection = described_class.new({})
+
+ connection.get('/users')
+ end
+ end
+
+ describe '#expired?' do
+ it 'calls connection.expired?' do
+ expect_any_instance_of(OAuth2::AccessToken).to receive(:expired?).and_return(true)
+
+ expect(described_class.new({}).expired?).to be_truthy
+ end
+ end
+
+ describe '#refresh!' do
+ it 'calls connection.refresh!' do
+ response = double(token: nil, expires_at: nil, expires_in: nil, refresh_token: nil)
+
+ expect_any_instance_of(OAuth2::AccessToken).to receive(:refresh!).and_return(response)
+
+ described_class.new({}).refresh!
+ end
+ end
+end
diff --git a/spec/lib/bitbucket/page_spec.rb b/spec/lib/bitbucket/page_spec.rb
new file mode 100644
index 00000000000..04d5a0470b1
--- /dev/null
+++ b/spec/lib/bitbucket/page_spec.rb
@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe Bitbucket::Page do
+ let(:response) { { 'values' => [{ 'username' => 'Ben' }], 'pagelen' => 2, 'next' => '' } }
+
+ before do
+ # Autoloading hack
+ Bitbucket::Representation::User.new({})
+ end
+
+ describe '#items' do
+ it 'returns collection of needed objects' do
+ page = described_class.new(response, :user)
+
+ expect(page.items.first).to be_a(Bitbucket::Representation::User)
+ expect(page.items.count).to eq(1)
+ end
+ end
+
+ describe '#attrs' do
+ it 'returns attributes' do
+ page = described_class.new(response, :user)
+
+ expect(page.attrs.keys).to include(:pagelen, :next)
+ end
+ end
+
+ describe '#next?' do
+ it 'returns true' do
+ page = described_class.new(response, :user)
+
+ expect(page.next?).to be_truthy
+ end
+
+ it 'returns false' do
+ response['next'] = nil
+ page = described_class.new(response, :user)
+
+ expect(page.next?).to be_falsey
+ end
+ end
+
+ describe '#next' do
+ it 'returns next attribute' do
+ page = described_class.new(response, :user)
+
+ expect(page.next).to eq('')
+ end
+ end
+end
diff --git a/spec/lib/bitbucket/paginator_spec.rb b/spec/lib/bitbucket/paginator_spec.rb
new file mode 100644
index 00000000000..2c972da682e
--- /dev/null
+++ b/spec/lib/bitbucket/paginator_spec.rb
@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Bitbucket::Paginator do
+ let(:last_page) { double(:page, next?: false, items: ['item_2']) }
+ let(:first_page) { double(:page, next?: true, next: last_page, items: ['item_1']) }
+
+ describe 'items' do
+ it 'return items and raises StopIteration in the end' do
+ paginator = described_class.new(nil, nil, nil)
+
+ allow(paginator).to receive(:fetch_next_page).and_return(first_page)
+ expect(paginator.items).to match(['item_1'])
+
+ allow(paginator).to receive(:fetch_next_page).and_return(last_page)
+ expect(paginator.items).to match(['item_2'])
+
+ allow(paginator).to receive(:fetch_next_page).and_return(nil)
+ expect{ paginator.items }.to raise_error(StopIteration)
+ end
+ end
+end
diff --git a/spec/lib/bitbucket/representation/comment_spec.rb b/spec/lib/bitbucket/representation/comment_spec.rb
new file mode 100644
index 00000000000..fec243a9f96
--- /dev/null
+++ b/spec/lib/bitbucket/representation/comment_spec.rb
@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Bitbucket::Representation::Comment do
+ describe '#author' do
+ it { expect(described_class.new('user' => { 'username' => 'Ben' }).author).to eq('Ben') }
+ it { expect(described_class.new({}).author).to be_nil }
+ end
+
+ describe '#note' do
+ it { expect(described_class.new('content' => { 'raw' => 'Text' }).note).to eq('Text') }
+ it { expect(described_class.new({}).note).to be_nil }
+ end
+
+ describe '#created_at' do
+ it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) }
+ end
+
+ describe '#updated_at' do
+ it { expect(described_class.new('updated_on' => Date.today).updated_at).to eq(Date.today) }
+ it { expect(described_class.new('created_on' => Date.today).updated_at).to eq(Date.today) }
+ end
+end
diff --git a/spec/lib/bitbucket/representation/issue_spec.rb b/spec/lib/bitbucket/representation/issue_spec.rb
new file mode 100644
index 00000000000..20f47224aa8
--- /dev/null
+++ b/spec/lib/bitbucket/representation/issue_spec.rb
@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Bitbucket::Representation::Issue do
+ describe '#iid' do
+ it { expect(described_class.new('id' => 1).iid).to eq(1) }
+ end
+
+ describe '#kind' do
+ it { expect(described_class.new('kind' => 'bug').kind).to eq('bug') }
+ end
+
+ describe '#milestone' do
+ it { expect(described_class.new({ 'milestone' => { 'name' => '1.0' } }).milestone).to eq('1.0') }
+ it { expect(described_class.new({}).milestone).to be_nil }
+ end
+
+ describe '#author' do
+ it { expect(described_class.new({ 'reporter' => { 'username' => 'Ben' } }).author).to eq('Ben') }
+ it { expect(described_class.new({}).author).to be_nil }
+ end
+
+ describe '#description' do
+ it { expect(described_class.new({ 'content' => { 'raw' => 'Text' } }).description).to eq('Text') }
+ it { expect(described_class.new({}).description).to be_nil }
+ end
+
+ describe '#state' do
+ it { expect(described_class.new({ 'state' => 'invalid' }).state).to eq('closed') }
+ it { expect(described_class.new({ 'state' => 'wontfix' }).state).to eq('closed') }
+ it { expect(described_class.new({ 'state' => 'resolved' }).state).to eq('closed') }
+ it { expect(described_class.new({ 'state' => 'duplicate' }).state).to eq('closed') }
+ it { expect(described_class.new({ 'state' => 'closed' }).state).to eq('closed') }
+ it { expect(described_class.new({ 'state' => 'opened' }).state).to eq('opened') }
+ end
+
+ describe '#title' do
+ it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') }
+ end
+
+ describe '#created_at' do
+ it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) }
+ end
+
+ describe '#updated_at' do
+ it { expect(described_class.new('edited_on' => Date.today).updated_at).to eq(Date.today) }
+ end
+end
diff --git a/spec/lib/bitbucket/representation/pull_request_comment_spec.rb b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb
new file mode 100644
index 00000000000..673dcf22ce8
--- /dev/null
+++ b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb
@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+describe Bitbucket::Representation::PullRequestComment do
+ describe '#iid' do
+ it { expect(described_class.new('id' => 1).iid).to eq(1) }
+ end
+
+ describe '#file_path' do
+ it { expect(described_class.new('inline' => { 'path' => '/path' }).file_path).to eq('/path') }
+ end
+
+ describe '#old_pos' do
+ it { expect(described_class.new('inline' => { 'from' => 3 }).old_pos).to eq(3) }
+ end
+
+ describe '#new_pos' do
+ it { expect(described_class.new('inline' => { 'to' => 3 }).new_pos).to eq(3) }
+ end
+
+ describe '#parent_id' do
+ it { expect(described_class.new({ 'parent' => { 'id' => 2 } }).parent_id).to eq(2) }
+ it { expect(described_class.new({}).parent_id).to be_nil }
+ end
+
+ describe '#inline?' do
+ it { expect(described_class.new('inline' => {}).inline?).to be_truthy }
+ it { expect(described_class.new({}).inline?).to be_falsey }
+ end
+
+ describe '#has_parent?' do
+ it { expect(described_class.new('parent' => {}).has_parent?).to be_truthy }
+ it { expect(described_class.new({}).has_parent?).to be_falsey }
+ end
+end
diff --git a/spec/lib/bitbucket/representation/pull_request_spec.rb b/spec/lib/bitbucket/representation/pull_request_spec.rb
new file mode 100644
index 00000000000..30453528be4
--- /dev/null
+++ b/spec/lib/bitbucket/representation/pull_request_spec.rb
@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Bitbucket::Representation::PullRequest do
+ describe '#iid' do
+ it { expect(described_class.new('id' => 1).iid).to eq(1) }
+ end
+
+ describe '#author' do
+ it { expect(described_class.new({ 'author' => { 'username' => 'Ben' } }).author).to eq('Ben') }
+ it { expect(described_class.new({}).author).to be_nil }
+ end
+
+ describe '#description' do
+ it { expect(described_class.new({ 'description' => 'Text' }).description).to eq('Text') }
+ it { expect(described_class.new({}).description).to be_nil }
+ end
+
+ describe '#state' do
+ it { expect(described_class.new({ 'state' => 'MERGED' }).state).to eq('merged') }
+ it { expect(described_class.new({ 'state' => 'DECLINED' }).state).to eq('closed') }
+ it { expect(described_class.new({}).state).to eq('opened') }
+ end
+
+ describe '#title' do
+ it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') }
+ end
+
+ describe '#source_branch_name' do
+ it { expect(described_class.new({ source: { branch: { name: 'feature' } } }.with_indifferent_access).source_branch_name).to eq('feature') }
+ it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_name).to be_nil }
+ end
+
+ describe '#source_branch_sha' do
+ it { expect(described_class.new({ source: { commit: { hash: 'abcd123' } } }.with_indifferent_access).source_branch_sha).to eq('abcd123') }
+ it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_sha).to be_nil }
+ end
+
+ describe '#target_branch_name' do
+ it { expect(described_class.new({ destination: { branch: { name: 'master' } } }.with_indifferent_access).target_branch_name).to eq('master') }
+ it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_name).to be_nil }
+ end
+
+ describe '#target_branch_sha' do
+ it { expect(described_class.new({ destination: { commit: { hash: 'abcd123' } } }.with_indifferent_access).target_branch_sha).to eq('abcd123') }
+ it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_sha).to be_nil }
+ end
+end
diff --git a/spec/lib/bitbucket/representation/user_spec.rb b/spec/lib/bitbucket/representation/user_spec.rb
new file mode 100644
index 00000000000..f79ff4edb7b
--- /dev/null
+++ b/spec/lib/bitbucket/representation/user_spec.rb
@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Bitbucket::Representation::User do
+ describe '#username' do
+ it 'returns correct value' do
+ user = described_class.new('username' => 'Ben')
+
+ expect(user.username).to eq('Ben')
+ end
+ end
+end
diff --git a/spec/lib/gitlab/bitbucket_import/client_spec.rb b/spec/lib/gitlab/bitbucket_import/client_spec.rb
deleted file mode 100644
index 7543c29bcc4..00000000000
--- a/spec/lib/gitlab/bitbucket_import/client_spec.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::BitbucketImport::Client, lib: true do
- include ImportSpecHelper
-
- let(:token) { '123456' }
- let(:secret) { 'secret' }
- let(:client) { Gitlab::BitbucketImport::Client.new(token, secret) }
-
- before do
- stub_omniauth_provider('bitbucket')
- end
-
- it 'all OAuth client options are symbols' do
- client.consumer.options.keys.each do |key|
- expect(key).to be_kind_of(Symbol)
- end
- end
-
- context 'issues' do
- let(:per_page) { 50 }
- let(:count) { 95 }
- let(:sample_issues) do
- issues = []
-
- count.times do |i|
- issues << { local_id: i }
- end
-
- issues
- end
- let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } }
- let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } }
- let(:project_id) { 'namespace/repo' }
-
- it 'retrieves issues over a number of pages' do
- stub_request(:get,
- "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0").
- to_return(status: 200,
- body: first_sample_data.to_json,
- headers: {})
-
- stub_request(:get,
- "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50").
- to_return(status: 200,
- body: second_sample_data.to_json,
- headers: {})
-
- issues = client.issues(project_id)
- expect(issues.count).to eq(95)
- end
- end
-
- context 'project import' do
- it 'calls .from_project with no errors' do
- project = create(:empty_project)
- project.import_url = "ssh://git@bitbucket.org/test/test.git"
- project.create_or_update_import_data(credentials:
- { user: "git",
- password: nil,
- bb_session: { bitbucket_access_token: "test",
- bitbucket_access_token_secret: "test" } })
-
- expect { described_class.from_project(project) }.not_to raise_error
- end
- end
-end
diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb
index aa00f32becb..53f3c73ade4 100644
--- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb
+++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb
@@ -18,15 +18,21 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
"closed" # undocumented status
]
end
+
let(:sample_issues_statuses) do
issues = []
statuses.map.with_index do |status, index|
issues << {
- local_id: index,
- status: status,
+ id: index,
+ state: status,
title: "Issue #{index}",
- content: "Some content to issue #{index}"
+ kind: 'bug',
+ content: {
+ raw: "Some content to issue #{index}",
+ markup: "markdown",
+ html: "Some content to issue #{index}"
+ }
}
end
@@ -34,14 +40,16 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
end
let(:project_identifier) { 'namespace/repo' }
+
let(:data) do
{
'bb_session' => {
- 'bitbucket_access_token' => "123456",
- 'bitbucket_access_token_secret' => "secret"
+ 'bitbucket_token' => "123456",
+ 'bitbucket_refresh_token' => "secret"
}
}
end
+
let(:project) do
create(
:project,
@@ -49,11 +57,13 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
import_data: ProjectImportData.new(credentials: data)
)
end
+
let(:importer) { Gitlab::BitbucketImport::Importer.new(project) }
+
let(:issues_statuses_sample_data) do
{
count: sample_issues_statuses.count,
- issues: sample_issues_statuses
+ values: sample_issues_statuses
}
end
@@ -61,26 +71,46 @@ describe Gitlab::BitbucketImport::Importer, lib: true do
before do
stub_request(
:get,
- "https://bitbucket.org/api/1.0/repositories/#{project_identifier}"
- ).to_return(status: 200, body: { has_issues: true }.to_json)
+ "https://api.bitbucket.org/2.0/repositories/#{project_identifier}"
+ ).to_return(status: 200,
+ headers: { "Content-Type" => "application/json" },
+ body: { has_issues: true, full_name: project_identifier }.to_json)
stub_request(
:get,
- "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues?limit=50&sort=utc_created_on&start=0"
- ).to_return(status: 200, body: issues_statuses_sample_data.to_json)
+ "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=50&sort=created_on"
+ ).to_return(status: 200,
+ headers: { "Content-Type" => "application/json" },
+ body: issues_statuses_sample_data.to_json)
+
+ stub_request(:get, "https://api.bitbucket.org/2.0/repositories/namespace/repo?pagelen=50&sort=created_on").
+ with(headers: { 'Accept' => '*/*', 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization' => 'Bearer', 'User-Agent' => 'Faraday v0.9.2' }).
+ to_return(status: 200,
+ body: "",
+ headers: {})
sample_issues_statuses.each_with_index do |issue, index|
stub_request(
:get,
- "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues/#{issue[:local_id]}/comments"
+ "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues/#{issue[:id]}/comments?pagelen=50&sort=created_on"
).to_return(
status: 200,
- body: [{ author_info: { username: "username" }, utc_created_on: index }].to_json
+ headers: { "Content-Type" => "application/json" },
+ body: { author_info: { username: "username" }, utc_created_on: index }.to_json
)
end
+
+ stub_request(
+ :get,
+ "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/pullrequests?pagelen=50&sort=created_on&state=ALL"
+ ).to_return(status: 200,
+ headers: { "Content-Type" => "application/json" },
+ body: {}.to_json)
end
it 'map statuses to open or closed' do
+ # HACK: Bitbucket::Representation.const_get('Issue') seems to return ::Issue without this
+ Bitbucket::Representation::Issue.new({})
importer.execute
expect(project.issues.where(state: "closed").size).to eq(5)
diff --git a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb
index e1c60e07b4d..b6d052a4612 100644
--- a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb
+++ b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb
@@ -2,14 +2,18 @@ require 'spec_helper'
describe Gitlab::BitbucketImport::ProjectCreator, lib: true do
let(:user) { create(:user) }
+
let(:repo) do
- {
- name: 'Vim',
- slug: 'vim',
- is_private: true,
- owner: "asd"
- }.with_indifferent_access
+ double(name: 'Vim',
+ slug: 'vim',
+ description: 'Test repo',
+ is_private: true,
+ owner: "asd",
+ full_name: 'Vim repo',
+ visibility_level: Gitlab::VisibilityLevel::PRIVATE,
+ clone_url: 'ssh://git@bitbucket.org/asd/vim.git')
end
+
let(:namespace){ create(:group, owner: user) }
let(:token) { "asdasd12345" }
let(:secret) { "sekrettt" }
@@ -22,7 +26,7 @@ describe Gitlab::BitbucketImport::ProjectCreator, lib: true do
it 'creates project' do
allow_any_instance_of(Project).to receive(:add_import_job)
- project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user, access_params)
+ project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, 'vim', namespace, user, access_params)
project = project_creator.execute
expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git")