diff options
| author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-12-21 12:09:06 +0100 |
|---|---|---|
| committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-12-21 12:09:06 +0100 |
| commit | 3deaf1342ec6bfe2122319329db8a8d4c8122e7e (patch) | |
| tree | 452388bb0542452f7b7864fe849059875766b2b2 /lib | |
| parent | 0dc5a21c9a7b78845c4e2ad1297a8a77c3dfb3e3 (diff) | |
| parent | 09b622f84c83ea65c773ca4eb53d6899dc1c4956 (diff) | |
| download | gitlab-ce-3deaf1342ec6bfe2122319329db8a8d4c8122e7e.tar.gz | |
Merge branch 'master' into auto-pipelines-vue
* master: (367 commits)
Set “Remove branch” button to default size
remove unused helper method
reduce common code even further to satisfy rake flay
remove button class size alteration from revert and cherry pick links
factor out common code to satisfy rake flay
homogenize revert and cherry-pick button styles generated by commits_helper
apply margin on alert banners only when there is one or more alerts
Rename MattermostNotificationService back to MattermostService
Rename SlackNotificationService back to SlackService
Fix stage and pipeline specs and rubocop offenses
Added QueryRecorder to test N+1 fix on Milestone#show
Use gitlab-workhorse 1.2.1
Make 'unmarked as WIP' message more consistent
Improve specs for Files API
Allow unauthenticated access to Repositories Files API GET endpoints
Add isolated view spec for pipeline stage partial
Move test for HTML stage endpoint to controller specs
Fix sizing of avatar circles; add border
Fix broken test
Fix broken test Changes after review
...
Conflicts:
app/assets/stylesheets/pages/pipelines.scss
app/controllers/projects/pipelines_controller.rb
app/views/projects/pipelines/index.html.haml
spec/features/projects/pipelines/pipelines_spec.rb
Diffstat (limited to 'lib')
68 files changed, 1490 insertions, 489 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index cec2702e44d..9d5adffd8f4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,8 @@ module API include APIGuard version 'v3', using: :path + before { allow_access_with_scope :api } + rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8cc7a26f1fa..df6db140d0e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -44,27 +47,60 @@ module API access_token = find_access_token return nil unless access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED + when AccessTokenValidationService::EXPIRED raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED + when AccessTokenValidationService::REVOKED raise RevokedError - when Oauth2::AccessTokenValidationService::VALID + when AccessTokenValidationService::VALID @current_user = User.find(access_token.resource_owner_id) end end + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + + return nil unless token_string.present? + + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + end + def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -72,10 +108,6 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods diff --git a/lib/api/files.rb b/lib/api/files.rb index 28f306e45f3..532a317c89e 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,8 +1,6 @@ module API # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index fbf538eda47..15ac29b2625 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -3,8 +3,6 @@ module API include Gitlab::Utils include Helpers::Pagination - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo @@ -303,7 +301,7 @@ module API private def private_token - params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] end def warden @@ -318,18 +316,11 @@ module API warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end - def find_user_by_private_token - token = private_token - return nil unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - def initial_current_user return @initial_current_user if defined?(@initial_current_user) - @initial_current_user ||= find_user_by_private_token - @initial_current_user ||= doorkeeper_guard + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) @initial_current_user ||= find_user_from_warden unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eb223c1101d..e8975eb57e0 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -52,6 +52,14 @@ module API :push_code ] end + + def parse_allowed_environment_variables + return if params[:env].blank? + + JSON.parse(params[:env]) + + rescue JSON::ParserError + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 7087ce11401..db2d18f935d 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -32,7 +32,11 @@ module API if wiki? Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + Gitlab::GitAccess.new(actor, + project, + protocol, + authentication_abilities: ssh_authentication_abilities, + env: parse_allowed_environment_variables) end access_status = access.check(params[:action], params[:changes]) diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index c287ee34a68..4ca6646a6f1 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -2,7 +2,6 @@ require 'mime/types' module API class Repositories < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do @@ -79,8 +78,6 @@ module API optional :format, type: String, desc: 'The archive format' end get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -96,7 +93,6 @@ module API requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' end get ':id/repository/compare' do - authorize! :download_code, user_project compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end @@ -105,8 +101,6 @@ module API success Entities::Contributor end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin present user_project.repository.contributors, with: Entities::Contributor diff --git a/lib/api/services.rb b/lib/api/services.rb index 59232c84c24..d11cdce4e18 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -378,7 +378,6 @@ module API desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' }, ], - 'mattermost-slash-commands' => [ { required: true, @@ -387,6 +386,14 @@ module API desc: 'The Mattermost token' } ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], 'pipelines-email' => [ { required: true, @@ -473,7 +480,7 @@ module API desc: 'The description of the tracker' } ], - 'slack-notification' => [ + 'slack' => [ { required: true, name: :webhook, @@ -493,7 +500,7 @@ module API desc: 'The channel name' } ], - 'mattermost-notification' => [ + 'mattermost' => [ { required: true, name: :webhook, diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 8a53d9c0095..e23f99256a5 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -8,6 +8,10 @@ module API gitlab_ci_ymls: { klass: Gitlab::Template::GitlabCiYmlTemplate, gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 } }.freeze PROJECT_TEMPLATE_REGEX = @@ -51,7 +55,7 @@ module API end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' - end + end get route do options = { featured: declared(params).popular.present? ? true : nil @@ -69,7 +73,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route, requirements: { name: /[\w\.-]+/ } do not_found!('License') unless Licensee::License.find(declared(params).name) @@ -78,7 +82,7 @@ module API present template, with: Entities::RepoLicense end end - + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| klass = properties[:klass] gitlab_version = properties[:gitlab_version] @@ -104,7 +108,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route do new_template = klass.find(declared(params).name) diff --git a/lib/api/users.rb b/lib/api/users.rb index c7db2d71017..4c22287b5c6 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -2,7 +2,10 @@ module API class Users < Grape::API include PaginationParams - before { authenticate! } + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do @@ -91,7 +94,7 @@ module API identity_attrs = params.slice(:provider, :extern_uid) confirm = params.delete(:confirm) - user = User.build_user(declared_params(include_missing: false)) + user = User.new(declared_params(include_missing: false)) user.skip_confirmation! unless confirm if identity_attrs.any? diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..7e55cf4deab --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..423eff8f2a5 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,71 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def has_wiki? + raw['has_wiki'] + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index ed87a2603e8..142bce82286 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -41,7 +41,7 @@ module Ci put ":id" do authenticate_runner! build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - forbidden!('Build has been erased!') if build.erased? + validate_build!(build) update_runner_info @@ -71,9 +71,7 @@ module Ci # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) - forbidden!('Build has been erased!') if build.erased? + authenticate_build!(build) error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] @@ -104,8 +102,7 @@ module Ci Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('build is not running') unless build.running? if params[:filesize] @@ -142,10 +139,8 @@ module Ci require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('Build is not running!') unless build.running? - forbidden!('Build has been erased!') if build.erased? artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) @@ -176,8 +171,7 @@ module Ci # GET /builds/:id/artifacts get ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -202,8 +196,7 @@ module Ci # DELETE /builds/:id/artifacts delete ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) build.erase_artifacts! end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e608f5f6cad..31fbd1da108 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,8 +13,19 @@ module Ci forbidden! unless current_runner end - def authenticate_build_token!(build) - forbidden! unless build_token_valid?(build) + def authenticate_build!(build) + validate_build!(build) do + forbidden! unless build_token_valid?(build) + end + end + + def validate_build!(build) + not_found! unless build + + yield if block_given? + + forbidden!('Project has been deleted!') unless build.project + forbidden!('Build has been erased!') if build.erased? end def runner_registration_token_valid? diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index fef652cb975..7463bd719d5 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -118,7 +118,7 @@ module Ci .merge(job_variables(name)) variables.map do |key, value| - { key: key, value: value, public: true } + { key: key.to_s, value: value, public: true } end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index aca5d0020cf..8dda65c71ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,10 @@ module Gitlab module Auth class MissingPersonalTokenError < StandardError; end + SCOPES = [:api, :read_user] + DEFAULT_SCOPES = [:api] + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -88,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -97,12 +101,27 @@ module Gitlab def personal_access_token_check(login, password) if login && password - user = User.find_by_personal_access_token(password) + token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + + if valid_personal_access_token?(token, validation) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end end end + def valid_oauth_token?(token) + token && token.accessible? && valid_api_token?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && valid_api_token?(token) + end + + def valid_api_token?(token) + AccessTokenValidationService.new(token).include_any_scope?(['api']) + end + def lfs_token_check(login, password) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..44323b47dca 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,247 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + include Gitlab::ShellAdapter + + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_wiki + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def identifier - project.import_source + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) end - def has_issues? - client.project(identifier)["has_issues"] + def repo + @repo ||= client.repo(project.import_source) end - def import_issues - issues = client.issues(identifier) + def import_wiki + return if project.wiki.repository_exists? - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + path_with_namespace = "#{project.path_with_namespace}.wiki" + import_url = project.import_url.sub(/\.git\z/, ".git/wiki") + gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + rescue StandardError => e + errors << { type: :wiki, errors: e.message } + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_issues + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - comments = client.issue_comments(identifier, issue["local_id"]) + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if comments.any? - body += @formatter.comments_header + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end + + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..d94f70fd1fb 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,17 +14,24 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data }, + skip_wiki: skip_wiki ).execute end + + private + + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index 25da8474e95..4fe53ce93a9 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -42,6 +42,10 @@ module Gitlab def find_by_iid(iid) collection.find_by(iid: iid) end + + def presenter + Gitlab::ChatCommands::Presenter.new + end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index b0d3fdbc48a..145086755e4 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -22,8 +22,6 @@ module Gitlab end end - private - def match_command match = nil service = available_commands.find do |klass| @@ -33,6 +31,8 @@ module Gitlab [service, match] end + private + def help_messages available_commands.map(&:help_message) end @@ -48,15 +48,15 @@ module Gitlab end def help(messages) - Mattermost::Presenter.help(messages, params[:command]) + presenter.help(messages, params[:command]) end def access_denied - Mattermost::Presenter.access_denied + presenter.access_denied end def present(resource) - Mattermost::Presenter.present(resource) + presenter.present(resource) end end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb index 0eed1fce0dc..6bb854dc080 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/chat_commands/deploy.rb @@ -4,7 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers def self.match(text) - /\Adeploy\s+(?<from>.*)\s+to+\s+(?<to>.*)\z/.match(text) + /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) end def self.help_message diff --git a/lib/mattermost/presenter.rb b/lib/gitlab/chat_commands/presenter.rb index 67eda983a74..caceaa25391 100644 --- a/lib/mattermost/presenter.rb +++ b/lib/gitlab/chat_commands/presenter.rb @@ -1,7 +1,7 @@ -module Mattermost - class Presenter - class << self - include Gitlab::Routing.url_helpers +module Gitlab + module ChatCommands + class Presenter + include Gitlab::Routing def authorize_chat_name(url) message = if url @@ -64,7 +64,7 @@ module Mattermost def single_resource(resource) return error(resource) if resource.errors.any? || !resource.persisted? - message = "### #{title(resource)}" + message = "#{title(resource)}:" message << "\n\n#{resource.description}" if resource.try(:description) in_channel_response(message) diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index cb1065223d4..3d203017d9f 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -3,11 +3,12 @@ module Gitlab class ChangeAccess attr_reader :user_access, :project - def initialize(change, user_access:, project:) + def initialize(change, user_access:, project:, env: {}) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project + @env = env end def exec @@ -68,7 +69,7 @@ module Gitlab end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 5fe86553bd0..de0c9049ebf 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,15 +1,20 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev) + def self.force_push?(project, oldrev, newrev, env: {}) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) - missed_ref.present? + missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute + + if exit_status == 0 + missed_ref.present? + else + raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." + end end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 5c506e6d59f..1bf949c96dd 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -17,6 +17,10 @@ module Gitlab 'icon_status_manual' end + def group + 'manual' + end + def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index f8ffa95cde4..e1dfdb76d41 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -17,6 +17,10 @@ module Gitlab 'icon_status_manual' end + def group + 'manual' + end + def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 46fef8262c1..73b6ab5a635 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -22,15 +22,8 @@ module Gitlab raise NotImplementedError end - # Deprecation warning: this method is here because we need to maintain - # backwards compatibility with legacy statuses. We often do something - # like "ci-status ci-status-#{status}" to set CSS class. - # - # `to_s` method should be renamed to `group` at some point, after - # phasing legacy satuses out. - # - def to_s - self.class.name.demodulize.downcase.underscore + def group + self.class.name.demodulize.underscore end def has_details? diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb index a7c98f9e909..24bf8b869e0 100644 --- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb +++ b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb @@ -17,7 +17,7 @@ module Gitlab 'icon_status_warning' end - def to_s + def group 'success_with_warnings' end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index c6bb8f9c8ed..9d142f1b82e 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -45,7 +45,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 85402c2a278..f586c5ab062 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -69,7 +69,7 @@ module Gitlab # This one might be controversial but so many reply lines have years, times and end with a colon. # Let's try it and see how well it works. break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) + (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb new file mode 100644 index 00000000000..79dd0cf7df2 --- /dev/null +++ b/lib/gitlab/git/rev_list.rb @@ -0,0 +1,42 @@ +module Gitlab + module Git + class RevList + attr_reader :project, :env + + ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze + + def initialize(oldrev, newrev, project:, env: nil) + @project = project + @env = env.presence || {} + @args = [Gitlab.config.git.bin_path, + "--git-dir=#{project.repository.path_to_repo}", + "rev-list", + "--max-count=1", + oldrev, + "^#{newrev}"] + end + + def execute + Gitlab::Popen.popen(@args, nil, parse_environment_variables) + end + + def valid? + environment_variables.all? do |(name, value)| + value.to_s.start_with?(project.repository.path_to_repo) + end + end + + private + + def parse_environment_variables + return {} unless valid? + + environment_variables + end + + def environment_variables + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db07b7c5fcc..c6b6efda360 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -17,12 +17,13 @@ module Gitlab attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, env: {}) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) + @env = env end def check(cmd, changes) @@ -103,7 +104,7 @@ module Gitlab end def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec + Checks::ChangeAccess.new(change, user_access: user_access, project: project, env: @env).exec end def protocol_allowed? diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 6dbae64a9fe..95dba9a327b 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,6 +15,10 @@ module Gitlab end end + def url + raw_data.url || '' + end + private def gitlab_user_id(github_id) diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 85df6547a67..ba869faa92e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,10 +4,12 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version if access_token ::Octokit.auto_paginate = false @@ -17,7 +19,7 @@ module Gitlab def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -64,6 +66,14 @@ module Gitlab private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 281b65bdeba..ec1318ab33c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,7 +3,7 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :errors, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @@ -11,12 +11,27 @@ module Gitlab @repo_url = project.import_url @errors = [] @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute @@ -35,7 +50,13 @@ module Gitlab import_comments(:issues) import_comments(:pull_requests) import_wiki - import_releases + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + handle_errors true @@ -44,7 +65,9 @@ module Gitlab private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil end def handle_errors @@ -60,9 +83,10 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - LabelFormatter.new(project, raw).create! + gh_label = LabelFormatter.new(project, raw) + gh_label.create! rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end end @@ -74,9 +98,10 @@ module Gitlab fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - MilestoneFormatter.new(project, raw).create! + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end end @@ -97,7 +122,7 @@ module Gitlab apply_labels(issuable, raw) rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end end @@ -106,18 +131,23 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = PullRequestFormatter.new(project, raw) - next unless pull_request.valid? + gh_pull_request = PullRequestFormatter.new(project, raw) + next unless gh_pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! - pull_request.create! + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure - clean_up_restored_branches(pull_request) + clean_up_restored_branches(gh_pull_request) end end end @@ -233,7 +263,7 @@ module Gitlab gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..256f360efc7 --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,60 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + def project_association + raise NotImplementedError + end + + def number + raw_data.number + end + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gitlab_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gitlab_author_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + if gitlab_author_id + body + else + formatter.author_line(author) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 887690bcc7c..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -24,59 +24,9 @@ module Gitlab :issues end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def pull_request? raw_data.pull_request.present? end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body - else - formatter.author_line(author) + body - end - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end end end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 401dd962521..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.number, + iid: number, project: project, title: raw_data.title, description: raw_data.description, @@ -19,7 +19,15 @@ module Gitlab end def find_condition - { iid: raw_data.number } + { iid: number } + end + + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index a2410068845..3f635be22ba 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,14 +1,15 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :name, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute @@ -19,7 +20,7 @@ module Gitlab description: repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - import_type: "github", + import_type: type, import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki @@ -29,7 +30,7 @@ module Gitlab private def import_url - repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b9a227fb11a..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -28,14 +28,6 @@ module Gitlab :merge_requests end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def valid? source_branch.valid? && target_branch.valid? end @@ -60,57 +52,15 @@ module Gitlab end end - def url - raw_data.url - end - private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - formatter.author_line(author) + body + super end end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c551321c18d..cda6ddf0443 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -120,7 +120,7 @@ module Gitlab members_mapper: members_mapper, user: @user, project_id: restored_project.id) - end + end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index a0e80fccad9..65b229ca8ff 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -40,6 +40,8 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return nil if unknown_service? + setup_models generate_imported_object @@ -99,6 +101,8 @@ module Gitlab def generate_imported_object if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') + @relation_hash.delete('token') + imported_object do |object| object.trace = trace object.commit_id = nil @@ -215,6 +219,11 @@ module Gitlab existing_object end end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 94261b7eeed..45958710c13 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,21 +7,38 @@ module Gitlab module ImportSources extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze + class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..288771c1c12 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token, ca_pem = nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:ca_pem] = ca_pem if ca_pem.present? + terminal + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index cc74bb29087..4bc5cda8cb5 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,13 +5,13 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil) + def popen(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end path ||= Dir.pwd - vars = { "PWD" => path } + vars['PWD'] = path options = { chdir: path } unless File.directory?(path) diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serialize/ci/variables.rb new file mode 100644 index 00000000000..3a9443bfcd9 --- /dev/null +++ b/lib/gitlab/serialize/ci/variables.rb @@ -0,0 +1,27 @@ +module Gitlab + module Serialize + module Ci + # This serializer could make sure our YAML variables' keys and values + # are always strings. This is more for legacy build data because + # from now on we convert them into strings before saving to database. + module Variables + extend self + + def load(string) + return unless string + + object = YAML.safe_load(string, [Symbol]) + + object.map do |variable| + variable[:key] = variable[:key].to_s + variable + end + end + + def dump(object) + YAML.dump(object) + end + end + end + end +end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb new file mode 100644 index 00000000000..d5d3e045a42 --- /dev/null +++ b/lib/gitlab/template/dockerfile_template.rb @@ -0,0 +1,30 @@ +module Gitlab + module Template + class DockerfileTemplate < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + 'Dockerfile' + end + + def categories + { + "General" => '' + } + end + + def base_dir + Rails.root.join('vendor/dockerfile') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index aeb1a26e1ba..d28bb583fe7 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -95,6 +95,19 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers] + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..fb8d7d97f8a --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,115 @@ +module Mattermost + class NoSessionError < StandardError; end + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + base_uri Settings.mattermost.host + + attr_accessor :current_resource_owner, :token + + def initialize(current_user) + @current_resource_owner = current_user + end + + def with_session + raise NoSessionError unless create + + begin + yield self + ensure + destroy + end + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + self.class.get(path, options.merge(headers: @headers)) + end + + def post(path, options = {}) + self.class.post(path, options.merge(headers: @headers)) + end + + private + + def create + return unless oauth_uri + return unless token_uri + + @token = request_token + @headers = { + Authorization: "Bearer #{@token}" + } + + @token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri = URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= + if oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + response = get(token_uri, follow_redirects: false) + + if 200 <= response.code && response.code < 400 + response.headers['token'] + end + end + end +end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..2f7c34a3f31 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -20,6 +20,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab { + default upgrade; + '' close; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -53,6 +58,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab; proxy_pass http://gitlab-workhorse; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..5661394058d 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -24,6 +24,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab_ssl { + default upgrade; + '' close; +} + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -98,6 +103,9 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab_ssl; + proxy_pass http://gitlab-workhorse; } diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake new file mode 100644 index 00000000000..c66a2a263dc --- /dev/null +++ b/lib/tasks/gitlab/ldap.rake @@ -0,0 +1,40 @@ +namespace :gitlab do + namespace :ldap do + desc 'GitLab | LDAP | Rename provider' + task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + old_provider = args[:old_provider] || + prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) + new_provider = args[:new_provider] || + prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue)) + puts '' # Add some separation in the output + + identities = Identity.where(provider: old_provider) + identity_count = identities.count + + if identities.empty? + puts "Found no user identities with '#{old_provider}' provider." + puts 'Please check the provider name and try again.' + exit 1 + end + + plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user') + + unless ENV['force'] == 'yes' + puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'" + puts 'If the new provider is incorrect, users will be unable to sign in' + ask_to_continue + puts '' + end + + updated_count = identities.update_all(provider: new_provider) + + if updated_count == identity_count + puts 'User identities were successfully updated'.color(:green) + else + plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user') + puts 'Some user identities could not be updated'.color(:red) + puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total" + end + end + end +end |