diff options
42 files changed, 1172 insertions, 510 deletions
@@ -22,7 +22,6 @@ gem 'doorkeeper', '~> 4.2.0' gem 'omniauth', '~> 1.3.1' gem 'omniauth-auth0', '~> 1.4.1' gem 'omniauth-azure-oauth2', '~> 0.0.6' -gem 'omniauth-bitbucket', '~> 0.0.2' gem 'omniauth-cas3', '~> 1.1.2' gem 'omniauth-facebook', '~> 4.0.0' gem 'omniauth-github', '~> 1.1.1' diff --git a/Gemfile.lock b/Gemfile.lock index 3de1a7cbf26..835cdc8cd84 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -412,10 +412,6 @@ GEM jwt (~> 1.0) omniauth (~> 1.0) omniauth-oauth2 (~> 1.1) - omniauth-bitbucket (0.0.2) - multi_json (~> 1.7) - omniauth (~> 1.1) - omniauth-oauth (~> 1.0) omniauth-cas3 (1.1.3) addressable (~> 2.3) nokogiri (~> 1.6.6) @@ -877,7 +873,6 @@ DEPENDENCIES omniauth (~> 1.3.1) omniauth-auth0 (~> 1.4.1) omniauth-azure-oauth2 (~> 0.0.6) - omniauth-bitbucket (~> 0.0.2) omniauth-cas3 (~> 1.1.2) omniauth-facebook (~> 4.0.0) omniauth-github (~> 1.1.1) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index bcc0b17bce2..4df80195ae1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -262,7 +262,7 @@ class ApplicationController < ActionController::Base end def bitbucket_import_configured? - Gitlab::OAuth::Provider.enabled?(:bitbucket) && Gitlab::BitbucketImport.public_key.present? + Gitlab::OAuth::Provider.enabled?(:bitbucket) end def google_code_import_enabled? diff --git a/app/controllers/import/bitbucket_controller.rb b/app/controllers/import/bitbucket_controller.rb index 6ea54744da8..b9cc6556140 100644 --- a/app/controllers/import/bitbucket_controller.rb +++ b/app/controllers/import/bitbucket_controller.rb @@ -2,50 +2,54 @@ class Import::BitbucketController < Import::BaseController before_action :verify_bitbucket_import_enabled before_action :bitbucket_auth, except: :callback - rescue_from OAuth::Error, with: :bitbucket_unauthorized - rescue_from Gitlab::BitbucketImport::Client::Unauthorized, with: :bitbucket_unauthorized + rescue_from OAuth2::Error, with: :bitbucket_unauthorized + rescue_from Bitbucket::Error::Unauthorized, with: :bitbucket_unauthorized def callback - request_token = session.delete(:oauth_request_token) - raise "Session expired!" if request_token.nil? + response = client.auth_code.get_token(params[:code], redirect_uri: callback_import_bitbucket_url) - request_token.symbolize_keys! - - access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url) - - session[:bitbucket_access_token] = access_token.token - session[:bitbucket_access_token_secret] = access_token.secret + session[:bitbucket_token] = response.token + session[:bitbucket_expires_at] = response.expires_at + session[:bitbucket_expires_in] = response.expires_in + session[:bitbucket_refresh_token] = response.refresh_token redirect_to status_import_bitbucket_url end def status - @repos = client.projects - @incompatible_repos = client.incompatible_projects + bitbucket_client = Bitbucket::Client.new(credentials) + repos = bitbucket_client.repos + + @repos, @incompatible_repos = repos.partition { |repo| repo.valid? } - @already_added_projects = current_user.created_projects.where(import_type: "bitbucket") + @already_added_projects = current_user.created_projects.where(import_type: 'bitbucket') already_added_projects_names = @already_added_projects.pluck(:import_source) - @repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" } + @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) } end def jobs - jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status]) - render json: jobs + render json: current_user.created_projects + .where(import_type: 'bitbucket') + .to_json(only: [:id, :import_status]) end def create + bitbucket_client = Bitbucket::Client.new(credentials) + @repo_id = params[:repo_id].to_s - repo = client.project(@repo_id.gsub('___', '/')) - @project_name = repo['slug'] - @target_namespace = find_or_create_namespace(repo['owner'], client.user['user']['username']) + name = @repo_id.gsub('___', '/') + repo = bitbucket_client.repo(name) + @project_name = params[:new_name].presence || repo.name - unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user, access_params).execute - render 'deploy_key' and return - end + repo_owner = repo.owner + repo_owner = current_user.username if repo_owner == bitbucket_client.user.username + @target_namespace = params[:new_namespace].presence || repo_owner + + namespace = find_or_create_namespace(@target_namespace, current_user) - if current_user.can?(:create_projects, @target_namespace) - @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @target_namespace, current_user, access_params).execute + if current_user.can?(:create_projects, namespace) + @project = Gitlab::BitbucketImport::ProjectCreator.new(repo, @project_name, namespace, current_user, credentials).execute else render 'unauthorized' end @@ -54,8 +58,15 @@ class Import::BitbucketController < Import::BaseController private def client - @client ||= Gitlab::BitbucketImport::Client.new(session[:bitbucket_access_token], - session[:bitbucket_access_token_secret]) + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys end def verify_bitbucket_import_enabled @@ -63,26 +74,23 @@ class Import::BitbucketController < Import::BaseController end def bitbucket_auth - if session[:bitbucket_access_token].blank? - go_to_bitbucket_for_permissions - end + go_to_bitbucket_for_permissions if session[:bitbucket_token].blank? end def go_to_bitbucket_for_permissions - request_token = client.request_token(callback_import_bitbucket_url) - session[:oauth_request_token] = request_token - - redirect_to client.authorize_url(request_token, callback_import_bitbucket_url) + redirect_to client.auth_code.authorize_url(redirect_uri: callback_import_bitbucket_url) end def bitbucket_unauthorized go_to_bitbucket_for_permissions end - def access_params + def credentials { - bitbucket_access_token: session[:bitbucket_access_token], - bitbucket_access_token_secret: session[:bitbucket_access_token_secret] + token: session[:bitbucket_token], + expires_at: session[:bitbucket_expires_at], + expires_in: session[:bitbucket_expires_in], + refresh_token: session[:bitbucket_refresh_token] } end end diff --git a/app/views/import/bitbucket/status.html.haml b/app/views/import/bitbucket/status.html.haml index f8b4b107513..ac09b71ae89 100644 --- a/app/views/import/bitbucket/status.html.haml +++ b/app/views/import/bitbucket/status.html.haml @@ -1,5 +1,6 @@ -- page_title "Bitbucket import" -- header_title "Projects", root_path +- page_title 'Bitbucket import' +- header_title 'Projects', root_path + %h3.page-title %i.fa.fa-bitbucket Import projects from Bitbucket @@ -10,13 +11,13 @@ %hr %p - if @incompatible_repos.any? - = button_tag class: "btn btn-import btn-success js-import-all" do + = button_tag class: 'btn btn-import btn-success js-import-all' do Import all compatible projects - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') - else - = button_tag class: "btn btn-success js-import-all" do + = button_tag class: 'btn btn-import btn-success js-import-all' do Import all projects - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') .table-responsive %table.table.import-jobs @@ -32,7 +33,7 @@ - @already_added_projects.each do |project| %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"} %td - = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: "_blank" + = link_to project.import_source, "https://bitbucket.org/#{project.import_source}", target: '_blank' %td = link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project] %td.job-status @@ -47,31 +48,41 @@ = project.human_import_status_name - @repos.each do |repo| - %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"} + %tr{id: "repo_#{repo.owner}___#{repo.slug}"} %td - = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank" + = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: "_blank" %td.import-target - = import_project_target(repo['owner'], repo['slug']) + %fieldset.row + .input-group + .project-path.input-group-btn + - if current_user.can_select_namespace? + - selected = params[:namespace_id] || :current_user + - opts = current_user.can_create_group? ? { extra_group: Group.new(name: repo.owner, path: repo.owner) } : {} + = select_tag :namespace_id, namespaces_options(selected, opts.merge({ display_path: true })), { class: 'select2 js-select-namespace', tabindex: 1 } + - else + = text_field_tag :path, current_user.namespace_path, class: "input-large form-control", tabindex: 1, disabled: true + %span.input-group-addon / + = text_field_tag :path, repo.name, class: "input-mini form-control", tabindex: 2, autofocus: true, required: true %td.import-actions.job-status - = button_tag class: "btn btn-import js-add-to-import" do + = button_tag class: 'btn btn-import js-add-to-import' do Import - = icon("spinner spin", class: "loading-icon") + = icon('spinner spin', class: 'loading-icon') - @incompatible_repos.each do |repo| - %tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"} + %tr{id: "repo_#{repo.owner}___#{repo.slug}"} %td - = link_to "#{repo["owner"]}/#{repo["slug"]}", "https://bitbucket.org/#{repo["owner"]}/#{repo["slug"]}", target: "_blank" + = link_to repo.full_name, "https://bitbucket.org/#{repo.full_name}", target: '_blank' %td.import-target %td.import-actions-job-status - = label_tag "Incompatible Project", nil, class: "label label-danger" + = label_tag 'Incompatible Project', nil, class: 'label label-danger' - if @incompatible_repos.any? %p One or more of your Bitbucket projects cannot be imported into GitLab directly because they use Subversion or Mercurial for version control, rather than Git. Please convert - = link_to "them to Git,", "https://www.atlassian.com/git/tutorials/migrating-overview" + = link_to 'them to Git,', 'https://www.atlassian.com/git/tutorials/migrating-overview' and go through the - = link_to "import flow", status_import_bitbucket_path, "data-no-turbolink" => "true" + = link_to 'import flow', status_import_bitbucket_path, 'data-no-turbolink' => 'true' again. .js-importer-status{ data: { jobs_import_path: "#{jobs_import_bitbucket_path}", import_path: "#{import_bitbucket_path}" } } diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 26c30e523a7..ab5a0561b8c 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -26,3 +26,9 @@ if Gitlab.config.omniauth.enabled end end end + +module OmniAuth + module Strategies + autoload :Bitbucket, Rails.root.join('lib', 'omniauth', 'strategies', 'bitbucket') + end +end diff --git a/config/initializers/public_key.rb b/config/initializers/public_key.rb deleted file mode 100644 index e4f09a2d020..00000000000 --- a/config/initializers/public_key.rb +++ /dev/null @@ -1,2 +0,0 @@ -path = File.expand_path("~/.ssh/bitbucket_rsa.pub") -Gitlab::BitbucketImport.public_key = File.read(path) if File.exist?(path) diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 9122dc62e39..9cdb101f457 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -44,14 +44,12 @@ you to use. And grant at least the following permissions: ``` - Account: Email - Repositories: Read, Admin + Account: Email, Read + Repositories: Read + Pull Requests: Read + Issues: Read ``` - >**Note:** - It may seem a little odd to giving GitLab admin permissions to repositories, - but this is needed in order for GitLab to be able to clone the repositories. -  1. Select **Save**. @@ -93,7 +91,8 @@ you to use. ```yaml - { name: 'bitbucket', app_id: 'BITBUCKET_APP_KEY', - app_secret: 'BITBUCKET_APP_SECRET' } + app_secret: 'BITBUCKET_APP_SECRET', + url: 'https://bitbucket.org/' } ``` --- @@ -112,91 +111,7 @@ well, the user will be returned to GitLab and will be signed in. ## Bitbucket project import -To allow projects to be imported directly into GitLab, Bitbucket requires two -extra setup steps compared to [GitHub](github.md) and [GitLab.com](gitlab.md). - -Bitbucket doesn't allow OAuth applications to clone repositories over HTTPS, and -instead requires GitLab to use SSH and identify itself using your GitLab -server's SSH key. - -To be able to access repositories on Bitbucket, GitLab will automatically -register your public key with Bitbucket as a deploy key for the repositories to -be imported. Your public key needs to be at `~/.ssh/bitbucket_rsa` which -translates to `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages and to -`/home/git/.ssh/bitbucket_rsa` for installations from source. - ---- - -Below are the steps that will allow GitLab to be able to import your projects -from Bitbucket. - -1. Make sure you [have enabled the Bitbucket OAuth support](#bitbucket-omniauth-provider). -1. Create a new SSH key with an **empty passphrase**: - - ```sh - sudo -u git -H ssh-keygen - ``` - - When asked to 'Enter file in which to save the key' enter: - `/var/opt/gitlab/.ssh/bitbucket_rsa` for Omnibus packages or - `/home/git/.ssh/bitbucket_rsa` for installations from source. The name is - important so make sure to get it right. - - > **Warning:** - This key must NOT be associated with ANY existing Bitbucket accounts. If it - is, the import will fail with an `Access denied! Please verify you can add - deploy keys to this repository.` error. - -1. Next, you need to to configure the SSH client to use your new key. Open the - SSH configuration file of the `git` user: - - ``` - # For Omnibus packages - sudo editor /var/opt/gitlab/.ssh/config - - # For installations from source - sudo editor /home/git/.ssh/config - ``` - -1. Add a host configuration for `bitbucket.org`: - - ```sh - Host bitbucket.org - IdentityFile ~/.ssh/bitbucket_rsa - User git - ``` - -1. Save the file and exit. -1. Manually connect to `bitbucket.org` over SSH, while logged in as the `git` - user that GitLab will use: - - ```sh - sudo -u git -H ssh bitbucket.org - ``` - - That step is performed because GitLab needs to connect to Bitbucket over SSH, - in order to add `bitbucket.org` to your GitLab server's known SSH hosts. - -1. Verify the RSA key fingerprint you'll see in the response matches the one - in the [Bitbucket documentation][bitbucket-docs] (the specific IP address - doesn't matter): - - ```sh - The authenticity of host 'bitbucket.org (104.192.143.1)' can't be established. - RSA key fingerprint is SHA256:zzXQOXSRBEiUtuE8AikJYKwbHaxvSc0ojez9YXaGp1A. - Are you sure you want to continue connecting (yes/no)? - ``` - -1. If the fingerprint matches, type `yes` to continue connecting and have - `bitbucket.org` be added to your known SSH hosts. After confirming you should - see a permission denied message. If you see an authentication successful - message you have done something wrong. The key you are using has already been - added to a Bitbucket account and will cause the import script to fail. Ensure - the key you are using CANNOT authenticate with Bitbucket. -1. Restart GitLab to allow it to find the new public key. - -Your GitLab server is now able to connect to Bitbucket over SSH. You should be -able to see the "Import projects from Bitbucket" option on the New Project page +You should be able to see the "Import projects from Bitbucket" option on the New Project page enabled. ## Acknowledgements diff --git a/doc/integration/img/bitbucket_oauth_settings_page.png b/doc/integration/img/bitbucket_oauth_settings_page.png Binary files differindex 8dbee9762d7..24acc6e1f5a 100644 --- a/doc/integration/img/bitbucket_oauth_settings_page.png +++ b/doc/integration/img/bitbucket_oauth_settings_page.png diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..5c2ef2a4509 --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + attr_reader :connection + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..c150a20761e --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..3c75e9368fa --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.dig('content', 'raw') + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..ffe8a65d839 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,49 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.dig('reporter', 'username') + end + + def description + raw.dig('content', 'raw') + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..e37c9a62c0e --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.dig('author', 'username') + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.dig('branch', 'name') + end + + def source_branch_sha + source_branch.dig('commit', 'hash') + end + + def target_branch_name + target_branch.dig('branch', 'name') + end + + def target_branch_sha + target_branch.dig('commit', 'hash') + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f3809fbcea --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.dig('parent', 'id') + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..8969ecd1c19 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,67 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..b6a0b122cdb 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,216 @@ module Gitlab module BitbucketImport class Importer + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + attr_reader :project, :client def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} end def execute - import_issues if has_issues? + import_issues + import_pull_requests true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + def gitlab_user_id(project, username) + if username + user = find_user(username) (user && user.id) || project.creator_id else project.creator_id end end - def identifier - project.import_source + def find_user(username) + User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) end - def has_issues? - client.project(identifier)["has_issues"] + def existing_gitlab_user?(username) + username && find_user(username) + end + + def repo + @repo ||= client.repo(project.import_source) end def import_issues - issues = client.issues(identifier) + return unless repo.issues_enabled? - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + create_labels - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter - end + client.issues(repo).each do |issue| + description = '' + description += @formatter.author_line(issue.author) unless existing_gitlab_user?(issue.author) + description += issue.description - body = @formatter.author_line(author) - body += issue["content"] + label_name = issue.kind - comments = client.issue_comments(identifier, issue["local_id"]) + issue = project.issues.create( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + created_at: issue.created_at, + updated_at: issue.updated_at + ) - if comments.any? - body += @formatter.comments_header + issue.labels << @labels[label_name] + + if issue.persisted? + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless existing_gitlab_user?(comment.author) + note += comment.note + + issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + end end + end + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Couldn't import record properly #{e.message}") + end - comments.each do |comment| - author = 'Anonymous' + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless existing_gitlab_user?(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue ActiveRecord::RecordInvalid + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request #{e.message}") + end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end + + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} + + children, parents = inline_comments.partition(&:has_parent?) + + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid pull request comment #{e.message}") + nil end + end + end - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error("Bitbucket importer ERROR in #{project.path_with_namespace}: Invalid standalone pull request comment #{e.message}") + nil + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..b34be272af3 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,15 +14,15 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(@session_data[:token]), + import_data: { credentials: session_data } ).execute end end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/spec/controllers/import/bitbucket_controller_spec.rb b/spec/controllers/import/bitbucket_controller_spec.rb index 1d3c9fbbe2f..ce7c0b334ee 100644 --- a/spec/controllers/import/bitbucket_controller_spec.rb +++ b/spec/controllers/import/bitbucket_controller_spec.rb @@ -6,11 +6,11 @@ describe Import::BitbucketController do let(:user) { create(:user) } let(:token) { "asdasd12345" } let(:secret) { "sekrettt" } - let(:access_params) { { bitbucket_access_token: token, bitbucket_access_token_secret: secret } } + let(:refresh_token) { SecureRandom.hex(15) } + let(:access_params) { { token: token, expires_at: nil, expires_in: nil, refresh_token: nil } } def assign_session_tokens - session[:bitbucket_access_token] = token - session[:bitbucket_access_token_secret] = secret + session[:bitbucket_token] = token end before do @@ -24,29 +24,36 @@ describe Import::BitbucketController do end it "updates access token" do - access_token = double(token: token, secret: secret) - allow_any_instance_of(Gitlab::BitbucketImport::Client). + expires_at = Time.now + 1.day + expires_in = 1.day + access_token = double(token: token, + secret: secret, + expires_at: expires_at, + expires_in: expires_in, + refresh_token: refresh_token) + allow_any_instance_of(OAuth2::Client). to receive(:get_token).and_return(access_token) stub_omniauth_provider('bitbucket') get :callback - expect(session[:bitbucket_access_token]).to eq(token) - expect(session[:bitbucket_access_token_secret]).to eq(secret) + expect(session[:bitbucket_token]).to eq(token) + expect(session[:bitbucket_refresh_token]).to eq(refresh_token) + expect(session[:bitbucket_expires_at]).to eq(expires_at) + expect(session[:bitbucket_expires_in]).to eq(expires_in) expect(controller).to redirect_to(status_import_bitbucket_url) end end describe "GET status" do before do - @repo = OpenStruct.new(slug: 'vim', owner: 'asd') + @repo = double(slug: 'vim', owner: 'asd', full_name: 'asd/vim', "valid?" => true) assign_session_tokens end it "assigns variables" do @project = create(:project, import_type: 'bitbucket', creator_id: user.id) - client = stub_client(projects: [@repo]) - allow(client).to receive(:incompatible_projects).and_return([]) + allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo]) get :status @@ -57,7 +64,7 @@ describe Import::BitbucketController do it "does not show already added project" do @project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim') - stub_client(projects: [@repo]) + allow_any_instance_of(Bitbucket::Client).to receive(:repos).and_return([@repo]) get :status @@ -70,19 +77,16 @@ describe Import::BitbucketController do let(:bitbucket_username) { user.username } let(:bitbucket_user) do - { user: { username: bitbucket_username } }.with_indifferent_access + double(username: bitbucket_username) end let(:bitbucket_repo) do - { slug: "vim", owner: bitbucket_username }.with_indifferent_access + double(slug: "vim", owner: bitbucket_username, name: 'vim') end before do - allow(Gitlab::BitbucketImport::KeyAdder). - to receive(:new).with(bitbucket_repo, user, access_params). - and_return(double(execute: true)) - - stub_client(user: bitbucket_user, project: bitbucket_repo) + allow_any_instance_of(Bitbucket::Client).to receive(:repo).and_return(bitbucket_repo) + allow_any_instance_of(Bitbucket::Client).to receive(:user).and_return(bitbucket_user) assign_session_tokens end @@ -90,7 +94,7 @@ describe Import::BitbucketController do context "when the Bitbucket user and GitLab user's usernames match" do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -102,7 +106,7 @@ describe Import::BitbucketController do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -114,7 +118,7 @@ describe Import::BitbucketController do let(:other_username) { "someone_else" } before do - bitbucket_repo["owner"] = other_username + allow(bitbucket_repo).to receive(:owner).and_return(other_username) end context "when a namespace with the Bitbucket user's username already exists" do @@ -123,7 +127,7 @@ describe Import::BitbucketController do context "when the namespace is owned by the GitLab user" do it "takes the existing namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, existing_namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, existing_namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -156,7 +160,7 @@ describe Import::BitbucketController do it "takes the new namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, an_instance_of(Group), user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, an_instance_of(Group), user, access_params). and_return(double(execute: true)) post :create, format: :js @@ -177,7 +181,7 @@ describe Import::BitbucketController do it "takes the current user's namespace" do expect(Gitlab::BitbucketImport::ProjectCreator). - to receive(:new).with(bitbucket_repo, user.namespace, user, access_params). + to receive(:new).with(bitbucket_repo, bitbucket_repo.name, user.namespace, user, access_params). and_return(double(execute: true)) post :create, format: :js diff --git a/spec/lib/bitbucket/collection_spec.rb b/spec/lib/bitbucket/collection_spec.rb new file mode 100644 index 00000000000..015a7f80e03 --- /dev/null +++ b/spec/lib/bitbucket/collection_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +# Emulates paginator. It returns 2 pages with results +class TestPaginator + def initialize + @current_page = 0 + end + + def items + @current_page += 1 + + raise StopIteration if @current_page > 2 + + ["result_1_page_#{@current_page}", "result_2_page_#{@current_page}"] + end +end + +describe Bitbucket::Collection do + it "iterates paginator" do + collection = described_class.new(TestPaginator.new) + + expect(collection.to_a).to match(["result_1_page_1", "result_2_page_1", "result_1_page_2", "result_2_page_2"]) + end +end diff --git a/spec/lib/bitbucket/connection_spec.rb b/spec/lib/bitbucket/connection_spec.rb new file mode 100644 index 00000000000..6be681a8b47 --- /dev/null +++ b/spec/lib/bitbucket/connection_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe Bitbucket::Connection do + describe '#get' do + it 'calls OAuth2::AccessToken::get' do + expect_any_instance_of(OAuth2::AccessToken).to receive(:get).and_return(double(parsed: true)) + + connection = described_class.new({}) + + connection.get('/users') + end + end + + describe '#expired?' do + it 'calls connection.expired?' do + expect_any_instance_of(OAuth2::AccessToken).to receive(:expired?).and_return(true) + + expect(described_class.new({}).expired?).to be_truthy + end + end + + describe '#refresh!' do + it 'calls connection.refresh!' do + response = double(token: nil, expires_at: nil, expires_in: nil, refresh_token: nil) + + expect_any_instance_of(OAuth2::AccessToken).to receive(:refresh!).and_return(response) + + described_class.new({}).refresh! + end + end +end diff --git a/spec/lib/bitbucket/page_spec.rb b/spec/lib/bitbucket/page_spec.rb new file mode 100644 index 00000000000..04d5a0470b1 --- /dev/null +++ b/spec/lib/bitbucket/page_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe Bitbucket::Page do + let(:response) { { 'values' => [{ 'username' => 'Ben' }], 'pagelen' => 2, 'next' => '' } } + + before do + # Autoloading hack + Bitbucket::Representation::User.new({}) + end + + describe '#items' do + it 'returns collection of needed objects' do + page = described_class.new(response, :user) + + expect(page.items.first).to be_a(Bitbucket::Representation::User) + expect(page.items.count).to eq(1) + end + end + + describe '#attrs' do + it 'returns attributes' do + page = described_class.new(response, :user) + + expect(page.attrs.keys).to include(:pagelen, :next) + end + end + + describe '#next?' do + it 'returns true' do + page = described_class.new(response, :user) + + expect(page.next?).to be_truthy + end + + it 'returns false' do + response['next'] = nil + page = described_class.new(response, :user) + + expect(page.next?).to be_falsey + end + end + + describe '#next' do + it 'returns next attribute' do + page = described_class.new(response, :user) + + expect(page.next).to eq('') + end + end +end diff --git a/spec/lib/bitbucket/paginator_spec.rb b/spec/lib/bitbucket/paginator_spec.rb new file mode 100644 index 00000000000..2c972da682e --- /dev/null +++ b/spec/lib/bitbucket/paginator_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe Bitbucket::Paginator do + let(:last_page) { double(:page, next?: false, items: ['item_2']) } + let(:first_page) { double(:page, next?: true, next: last_page, items: ['item_1']) } + + describe 'items' do + it 'return items and raises StopIteration in the end' do + paginator = described_class.new(nil, nil, nil) + + allow(paginator).to receive(:fetch_next_page).and_return(first_page) + expect(paginator.items).to match(['item_1']) + + allow(paginator).to receive(:fetch_next_page).and_return(last_page) + expect(paginator.items).to match(['item_2']) + + allow(paginator).to receive(:fetch_next_page).and_return(nil) + expect{ paginator.items }.to raise_error(StopIteration) + end + end +end diff --git a/spec/lib/bitbucket/representation/comment_spec.rb b/spec/lib/bitbucket/representation/comment_spec.rb new file mode 100644 index 00000000000..5864193cbfc --- /dev/null +++ b/spec/lib/bitbucket/representation/comment_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Bitbucket::Representation::Comment do + describe '#author' do + it { expect(described_class.new('user' => { 'username' => 'Ben' }).author).to eq('Ben') } + it { expect(described_class.new({}).author).to eq('Anonymous') } + end + + describe '#note' do + it { expect(described_class.new('content' => { 'raw' => 'Text' }).note).to eq('Text') } + it { expect(described_class.new({}).note).to be_nil } + end + + describe '#created_at' do + it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) } + end + + describe '#updated_at' do + it { expect(described_class.new('updated_on' => Date.today).updated_at).to eq(Date.today) } + it { expect(described_class.new('created_on' => Date.today).updated_at).to eq(Date.today) } + end +end diff --git a/spec/lib/bitbucket/representation/issue_spec.rb b/spec/lib/bitbucket/representation/issue_spec.rb new file mode 100644 index 00000000000..56deae63bbc --- /dev/null +++ b/spec/lib/bitbucket/representation/issue_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Bitbucket::Representation::Issue do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#kind' do + it { expect(described_class.new('kind' => 'bug').kind).to eq('bug') } + end + + describe '#author' do + it { expect(described_class.new({ 'reporter' => { 'username' => 'Ben' }}).author).to eq('Ben') } + it { expect(described_class.new({}).author).to eq('Anonymous') } + end + + describe '#description' do + it { expect(described_class.new({ 'content' => { 'raw' => 'Text' }}).description).to eq('Text') } + it { expect(described_class.new({}).description).to be_nil } + end + + describe '#state' do + it { expect(described_class.new({ 'state' => 'invalid' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'wontfix' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'resolved' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'duplicate' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'closed' }).state).to eq('closed') } + it { expect(described_class.new({ 'state' => 'opened' }).state).to eq('opened') } + end + + describe '#title' do + it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') } + end + + describe '#created_at' do + it { expect(described_class.new('created_on' => Date.today).created_at).to eq(Date.today) } + end + + describe '#updated_at' do + it { expect(described_class.new('edited_on' => Date.today).updated_at).to eq(Date.today) } + end +end diff --git a/spec/lib/bitbucket/representation/pull_request_comment_spec.rb b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb new file mode 100644 index 00000000000..8377f0540cd --- /dev/null +++ b/spec/lib/bitbucket/representation/pull_request_comment_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe Bitbucket::Representation::PullRequestComment do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#file_path' do + it { expect(described_class.new('inline' => { 'path' => '/path' }).file_path).to eq('/path') } + end + + describe '#old_pos' do + it { expect(described_class.new('inline' => { 'from' => 3 }).old_pos).to eq(3) } + end + + describe '#new_pos' do + it { expect(described_class.new('inline' => { 'to' => 3 }).new_pos).to eq(3) } + end + + + describe '#parent_id' do + it { expect(described_class.new({ 'parent' => { 'id' => 2 }}).parent_id).to eq(2) } + it { expect(described_class.new({}).parent_id).to be_nil } + end + + describe '#inline?' do + it { expect(described_class.new('inline' => {}).inline?).to be_truthy } + it { expect(described_class.new({}).inline?).to be_falsey } + end + + describe '#has_parent?' do + it { expect(described_class.new('parent' => {}).has_parent?).to be_truthy } + it { expect(described_class.new({}).has_parent?).to be_falsey } + end +end diff --git a/spec/lib/bitbucket/representation/pull_request_spec.rb b/spec/lib/bitbucket/representation/pull_request_spec.rb new file mode 100644 index 00000000000..661422efddf --- /dev/null +++ b/spec/lib/bitbucket/representation/pull_request_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +describe Bitbucket::Representation::PullRequest do + describe '#iid' do + it { expect(described_class.new('id' => 1).iid).to eq(1) } + end + + describe '#author' do + it { expect(described_class.new({ 'author' => { 'username' => 'Ben' }}).author).to eq('Ben') } + it { expect(described_class.new({}).author).to eq('Anonymous') } + end + + describe '#description' do + it { expect(described_class.new({ 'description' => 'Text' }).description).to eq('Text') } + it { expect(described_class.new({}).description).to be_nil } + end + + describe '#state' do + it { expect(described_class.new({ 'state' => 'MERGED' }).state).to eq('merged') } + it { expect(described_class.new({ 'state' => 'DECLINED' }).state).to eq('closed') } + it { expect(described_class.new({}).state).to eq('opened') } + end + + describe '#title' do + it { expect(described_class.new('title' => 'Issue').title).to eq('Issue') } + end + + describe '#source_branch_name' do + it { expect(described_class.new({ source: { branch: { name: 'feature' } } }.with_indifferent_access).source_branch_name).to eq('feature') } + it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_name).to be_nil } + end + + describe '#source_branch_sha' do + it { expect(described_class.new({ source: { commit: { hash: 'abcd123' } } }.with_indifferent_access).source_branch_sha).to eq('abcd123') } + it { expect(described_class.new({ source: {} }.with_indifferent_access).source_branch_sha).to be_nil } + end + + describe '#target_branch_name' do + it { expect(described_class.new({ destination: { branch: { name: 'master' } } }.with_indifferent_access).target_branch_name).to eq('master') } + it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_name).to be_nil } + end + + describe '#target_branch_sha' do + it { expect(described_class.new({ destination: { commit: { hash: 'abcd123' } } }.with_indifferent_access).target_branch_sha).to eq('abcd123') } + it { expect(described_class.new({ destination: {} }.with_indifferent_access).target_branch_sha).to be_nil } + end +end diff --git a/spec/lib/bitbucket/representation/user_spec.rb b/spec/lib/bitbucket/representation/user_spec.rb new file mode 100644 index 00000000000..f79ff4edb7b --- /dev/null +++ b/spec/lib/bitbucket/representation/user_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe Bitbucket::Representation::User do + describe '#username' do + it 'returns correct value' do + user = described_class.new('username' => 'Ben') + + expect(user.username).to eq('Ben') + end + end +end diff --git a/spec/lib/gitlab/bitbucket_import/client_spec.rb b/spec/lib/gitlab/bitbucket_import/client_spec.rb deleted file mode 100644 index 7543c29bcc4..00000000000 --- a/spec/lib/gitlab/bitbucket_import/client_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'spec_helper' - -describe Gitlab::BitbucketImport::Client, lib: true do - include ImportSpecHelper - - let(:token) { '123456' } - let(:secret) { 'secret' } - let(:client) { Gitlab::BitbucketImport::Client.new(token, secret) } - - before do - stub_omniauth_provider('bitbucket') - end - - it 'all OAuth client options are symbols' do - client.consumer.options.keys.each do |key| - expect(key).to be_kind_of(Symbol) - end - end - - context 'issues' do - let(:per_page) { 50 } - let(:count) { 95 } - let(:sample_issues) do - issues = [] - - count.times do |i| - issues << { local_id: i } - end - - issues - end - let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } } - let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } } - let(:project_id) { 'namespace/repo' } - - it 'retrieves issues over a number of pages' do - stub_request(:get, - "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0"). - to_return(status: 200, - body: first_sample_data.to_json, - headers: {}) - - stub_request(:get, - "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50"). - to_return(status: 200, - body: second_sample_data.to_json, - headers: {}) - - issues = client.issues(project_id) - expect(issues.count).to eq(95) - end - end - - context 'project import' do - it 'calls .from_project with no errors' do - project = create(:empty_project) - project.import_url = "ssh://git@bitbucket.org/test/test.git" - project.create_or_update_import_data(credentials: - { user: "git", - password: nil, - bb_session: { bitbucket_access_token: "test", - bitbucket_access_token_secret: "test" } }) - - expect { described_class.from_project(project) }.not_to raise_error - end - end -end diff --git a/spec/lib/gitlab/bitbucket_import/importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importer_spec.rb index aa00f32becb..353312675d6 100644 --- a/spec/lib/gitlab/bitbucket_import/importer_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/importer_spec.rb @@ -18,15 +18,21 @@ describe Gitlab::BitbucketImport::Importer, lib: true do "closed" # undocumented status ] end + let(:sample_issues_statuses) do issues = [] statuses.map.with_index do |status, index| issues << { - local_id: index, - status: status, + id: index, + state: status, title: "Issue #{index}", - content: "Some content to issue #{index}" + kind: 'bug', + content: { + raw: "Some content to issue #{index}", + markup: "markdown", + html: "Some content to issue #{index}" + } } end @@ -34,14 +40,16 @@ describe Gitlab::BitbucketImport::Importer, lib: true do end let(:project_identifier) { 'namespace/repo' } + let(:data) do { 'bb_session' => { - 'bitbucket_access_token' => "123456", - 'bitbucket_access_token_secret' => "secret" + 'bitbucket_token' => "123456", + 'bitbucket_refresh_token' => "secret" } } end + let(:project) do create( :project, @@ -49,11 +57,13 @@ describe Gitlab::BitbucketImport::Importer, lib: true do import_data: ProjectImportData.new(credentials: data) ) end + let(:importer) { Gitlab::BitbucketImport::Importer.new(project) } + let(:issues_statuses_sample_data) do { count: sample_issues_statuses.count, - issues: sample_issues_statuses + values: sample_issues_statuses } end @@ -61,26 +71,46 @@ describe Gitlab::BitbucketImport::Importer, lib: true do before do stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}" - ).to_return(status: 200, body: { has_issues: true }.to_json) + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: { has_issues: true, full_name: project_identifier }.to_json) stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues?limit=50&sort=utc_created_on&start=0" - ).to_return(status: 200, body: issues_statuses_sample_data.to_json) + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues?pagelen=50&sort=created_on" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: issues_statuses_sample_data.to_json) + + stub_request(:get, "https://api.bitbucket.org/2.0/repositories/namespace/repo?pagelen=50&sort=created_on"). + with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer', 'User-Agent'=>'Faraday v0.9.2'}). + to_return(:status => 200, + :body => "", + :headers => {}) sample_issues_statuses.each_with_index do |issue, index| stub_request( :get, - "https://bitbucket.org/api/1.0/repositories/#{project_identifier}/issues/#{issue[:local_id]}/comments" + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/issues/#{issue[:id]}/comments?pagelen=50&sort=created_on" ).to_return( status: 200, - body: [{ author_info: { username: "username" }, utc_created_on: index }].to_json + headers: { "Content-Type" => "application/json" }, + body: { author_info: { username: "username" }, utc_created_on: index }.to_json ) end + + stub_request( + :get, + "https://api.bitbucket.org/2.0/repositories/#{project_identifier}/pullrequests?pagelen=50&sort=created_on&state=ALL" + ).to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: {}.to_json) end it 'map statuses to open or closed' do + # HACK: Bitbucket::Representation.const_get('Issue') seems to return ::Issue without this + Bitbucket::Representation::Issue.new({}) importer.execute expect(project.issues.where(state: "closed").size).to eq(5) diff --git a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb index e1c60e07b4d..b6d052a4612 100644 --- a/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/project_creator_spec.rb @@ -2,14 +2,18 @@ require 'spec_helper' describe Gitlab::BitbucketImport::ProjectCreator, lib: true do let(:user) { create(:user) } + let(:repo) do - { - name: 'Vim', - slug: 'vim', - is_private: true, - owner: "asd" - }.with_indifferent_access + double(name: 'Vim', + slug: 'vim', + description: 'Test repo', + is_private: true, + owner: "asd", + full_name: 'Vim repo', + visibility_level: Gitlab::VisibilityLevel::PRIVATE, + clone_url: 'ssh://git@bitbucket.org/asd/vim.git') end + let(:namespace){ create(:group, owner: user) } let(:token) { "asdasd12345" } let(:secret) { "sekrettt" } @@ -22,7 +26,7 @@ describe Gitlab::BitbucketImport::ProjectCreator, lib: true do it 'creates project' do allow_any_instance_of(Project).to receive(:add_import_job) - project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user, access_params) + project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, 'vim', namespace, user, access_params) project = project_creator.execute expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git") |
