diff options
author | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2016-09-21 12:22:02 +0300 |
---|---|---|
committer | Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> | 2016-09-21 12:22:02 +0300 |
commit | c2a7e7b8ac283f19ef324e93380ea1c802c6e7d0 (patch) | |
tree | f52e2189b463423b9cf0a9dbae159df36eed0da5 /lib | |
parent | 8799103037873b9009a6cbfe5fd6077202531ef1 (diff) | |
parent | 6df3dd9d5ed6c16d60f560c906dded888d845953 (diff) | |
download | gitlab-ce-c2a7e7b8ac283f19ef324e93380ea1c802c6e7d0.tar.gz |
Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce
Diffstat (limited to 'lib')
31 files changed, 632 insertions, 164 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index d02b469dac8..29a97ccbd75 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -20,7 +20,7 @@ module API access_requesters = paginate(source.requesters.includes(:user)) - present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters + present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end # Request access to the group/project diff --git a/lib/api/entities.rb b/lib/api/entities.rb index bfee4b6c752..92a6f29adb0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -100,22 +100,23 @@ module API SharedGroup.represent(project.project_group_links.all, options) end expose :only_allow_merge_if_build_succeeds + expose :request_access_enabled end class Member < UserBasic expose :access_level do |user, options| - member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member = options[:member] || options[:source].members.find_by(user_id: user.id) member.access_level end expose :expires_at do |user, options| - member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member = options[:member] || options[:source].members.find_by(user_id: user.id) member.expires_at end end class AccessRequester < UserBasic expose :requested_at do |user, options| - access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id } + access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id) access_requester.requested_at end end @@ -125,6 +126,7 @@ module API expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url + expose :request_access_enabled end class GroupDetail < Group diff --git a/lib/api/files.rb b/lib/api/files.rb index c1d86f313b0..96510e651a3 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -11,14 +11,16 @@ module API target_branch: attrs[:branch_name], commit_message: attrs[:commit_message], file_content: attrs[:content], - file_content_encoding: attrs[:encoding] + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] } end def commit_response(attrs) { file_path: attrs[:file_path], - branch_name: attrs[:branch_name], + branch_name: attrs[:branch_name] } end end @@ -96,7 +98,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -122,7 +124,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -149,7 +151,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name] result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 60ac9bdfa33..953fa474e88 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -23,18 +23,19 @@ module API # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # POST /groups post do authorize! :create_group required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] @group = Group.new(attrs) if @group.save @@ -48,18 +49,19 @@ module API # Update group. Available only for users who can administrate groups. # # Parameters: - # id (required) - The ID of a group - # path (optional) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # PUT /groups/:id put ':id' do group = find_group(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] if ::Groups::UpdateService.new(group, current_user, attrs).execute present group, with: Entities::GroupDetail diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 6e6efece7c4..090d04544da 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -35,6 +35,14 @@ module API Project.find_with_namespace(project_path) end end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end end post "/allowed" do @@ -51,9 +59,9 @@ module API access = if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol) + Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol) + Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) end access_status = access.check(params[:action], params[:changes]) @@ -74,6 +82,19 @@ module API response end + post "/lfs_authenticate" do + status 200 + + key = Key.find(params[:key_id]) + token_handler = Gitlab::LfsToken.new(key) + + { + username: token_handler.actor_name, + lfs_token: token_handler.generate, + repository_http_path: project.http_url_to_repo + } + end + get "/merge_request_urls" do ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end diff --git a/lib/api/members.rb b/lib/api/members.rb index 94c16710d9a..37f0a6512f4 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -18,11 +18,11 @@ module API get ":id/members" do source = find_source(source_type, params[:id]) - members = source.members.includes(:user) - members = members.joins(:user).merge(User.search(params[:query])) if params[:query] - members = paginate(members) + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] + users = paginate(users) - present members.map(&:user), with: Entities::Member, members: members + present users, with: Entities::Member, source: source end # Get a group/project member diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8bfa998dc53..c5c214d4d13 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -83,12 +83,12 @@ module API opts[:created_at] = params[:created_at] end - @note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::const_get(note.class.name) else - not_found!("Note #{@note.errors.messages}") + not_found!("Note #{note.errors.messages}") end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 644d836ed0b..5eb83c2c8f8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -91,8 +91,8 @@ module API # Create new project # # Parameters: - # name (required) - name for new project - # description (optional) - short project description + # name (required) - name for new project + # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -100,33 +100,35 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default + # namespace_id (optional) - defaults to user namespace + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) - 0 by default # import_url (optional) # public_builds (optional) # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects post do required_attributes! [:name] - attrs = attributes_for_keys [:name, - :path, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, :namespace_id, + :only_allow_merge_if_build_succeeds, + :path, :public, - :visibility_level, - :import_url, :public_builds, - :only_allow_merge_if_build_succeeds, - :lfs_enabled] + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :visibility_level, + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -143,10 +145,10 @@ module API # Create new project for a specified user. Only available to admin users. # # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default + # user_id (required) - The ID of a user + # name (required) - name for new project + # description (optional) - short project description + # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -154,31 +156,33 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 + # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) # public_builds (optional) # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects/user/:user_id post "user/:user_id" do authenticated_as_admin! user = User.find(params[:user_id]) - attrs = attributes_for_keys [:name, - :description, + attrs = attributes_for_keys [:builds_enabled, :default_branch, + :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, :public, - :visibility_level, - :import_url, :public_builds, - :only_allow_merge_if_build_succeeds, - :lfs_enabled] + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :visibility_level, + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -242,22 +246,23 @@ module API # Example Request # PUT /projects/:id put ':id' do - attrs = attributes_for_keys [:name, - :path, - :description, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :default_branch, + :description, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, + :path, :public, - :visibility_level, :public_builds, - :only_allow_merge_if_build_succeeds, - :lfs_enabled] + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :visibility_level, + :wiki_enabled] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..66c05773b68 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -15,6 +15,15 @@ module Ci expose :filename, :size end + class BuildOptions < Grape::Entity + expose :image + expose :services + expose :artifacts + expose :cache + expose :dependencies + expose :after_script + end + class Build < Grape::Entity expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index ba80c89df78..23353c62885 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -14,12 +14,20 @@ module Ci end def authenticate_build_token!(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - forbidden! unless token && build.valid_token?(token) + forbidden! unless build_token_valid?(build) end def runner_registration_token_valid? - params[:token] == current_application_settings.runners_registration_token + ActiveSupport::SecurityUtils.variable_size_secure_compare( + params[:token], + current_application_settings.runners_registration_token) + end + + def build_token_valid?(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + + # We require to also check `runners_token` to maintain compatibility with old version of runners + token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end def update_runner_last_contact(save: true) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index caa815f720f..0369e80312a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -60,7 +60,7 @@ module Ci name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], + environment: job[:environment_name], yaml_variables: yaml_variables(name), options: { image: job[:image], @@ -69,6 +69,7 @@ module Ci cache: job[:cache], dependencies: job[:dependencies], after_script: job[:after_script], + environment: job[:environment], }.compact } end diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb new file mode 100644 index 00000000000..997377abc55 --- /dev/null +++ b/lib/ci/mask_secret.rb @@ -0,0 +1,10 @@ +module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end +end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb new file mode 100644 index 00000000000..7b1533d0d32 --- /dev/null +++ b/lib/expand_variables.rb @@ -0,0 +1,17 @@ +module ExpandVariables + class << self + def expand(value, variables) + # Convert hash array to variables + if variables.is_a?(Array) + variables = variables.reduce({}) do |hash, variable| + hash[variable[:key]] = variable[:value] + hash + end + end + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables[$1 || $2] + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 91f0270818a..7c0f2115d43 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,21 +1,22 @@ module Gitlab module Auth - Result = Struct.new(:user, :type) + class MissingPersonalTokenError < StandardError; end class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? - result = Result.new + result = + service_request_check(login, password, project) || + build_access_token_check(login, password) || + user_with_password_for_git(login, password) || + oauth_access_token_check(login, password) || + lfs_token_check(login, password) || + personal_access_token_check(login, password) || + Gitlab::Auth::Result.new - if valid_ci_request?(login, password, project) - result.type = :ci - else - result = populate_result(login, password) - end + rate_limit!(ip, success: result.success?, login: login) - success = result.user.present? || [:ci, :missing_personal_token].include?(result.type) - rate_limit!(ip, success: success, login: login) result end @@ -57,44 +58,31 @@ module Gitlab private - def valid_ci_request?(login, password, project) + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) - return false unless project && matched_login.present? + return unless project && matched_login.present? underscored_service = matched_login['service'].underscore - if underscored_service == 'gitlab_ci' - project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) + if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. service = project.public_send("#{underscored_service}_service") - service && service.activated? && service.valid_token?(password) - end - end - - def populate_result(login, password) - result = - user_with_password_for_git(login, password) || - oauth_access_token_check(login, password) || - personal_access_token_check(login, password) - - if result - result.type = nil unless result.user - - if result.user && result.user.two_factor_enabled? && result.type == :gitlab_or_ldap - result.type = :missing_personal_token + if service && service.activated? && service.valid_token?(password) + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) end end - - result || Result.new end def user_with_password_for_git(login, password) user = find_with_user_password(login, password) - Result.new(user, :gitlab_or_ldap) if user + return unless user + + raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + + Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end def oauth_access_token_check(login, password) @@ -102,7 +90,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if token && token.accessible? user = User.find_by(id: token.resource_owner_id) - Result.new(user, :oauth) + Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end end end @@ -111,9 +99,76 @@ module Gitlab if login && password user = User.find_by_personal_access_token(password) validation = User.by_login(login) - Result.new(user, :personal_token) if user == validation + Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + end + end + + def lfs_token_check(login, password) + deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) + + actor = + if deploy_key_matches + DeployKey.find(deploy_key_matches[1]) + else + User.by_login(login) + end + + return unless actor + + token_handler = Gitlab::LfsToken.new(actor) + + authentication_abilities = + if token_handler.user? + full_authentication_abilities + else + read_authentication_abilities + end + + Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.value, password) + end + + def build_access_token_check(login, password) + return unless login == 'gitlab-ci-token' + return unless password + + build = ::Ci::Build.running.find_by_token(password) + return unless build + return unless build.project.builds_enabled? + + if build.user + # If user is assigned to build, use restricted credentials of user + Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) + else + # Otherwise use generic CI credentials (backward compatibility) + Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities) end end + + public + + def build_authentication_abilities + [ + :read_project, + :build_download_code, + :build_read_container_image, + :build_create_container_image + ] + end + + def read_authentication_abilities + [ + :read_project, + :download_code, + :read_container_image + ] + end + + def full_authentication_abilities + read_authentication_abilities + [ + :push_code, + :create_container_image + ] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb new file mode 100644 index 00000000000..6be7f690676 --- /dev/null +++ b/lib/gitlab/auth/result.rb @@ -0,0 +1,21 @@ +module Gitlab + module Auth + Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + def ci?(for_project) + type == :ci && + project && + project == for_project + end + + def lfs_deploy_token?(for_project) + type == :lfs_deploy_token && + actor && + actor.projects.include?(for_project) + end + + def success? + actor.present? || type == :ci + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index c412249a01e..79eac66b364 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -6,7 +6,12 @@ module Gitlab KeyAdder = Struct.new(:io) do def add_key(id, key) - key.gsub!(/[[:space:]]+/, ' ').strip! + key = Gitlab::Shell.strip_key(key) + # Newline and tab are part of the 'protocol' used to transmit id+key to the other end + if key.include?("\t") || key.include?("\n") + raise Error.new("Invalid key: #{key.inspect}") + end + io.puts("#{id}\t#{key}") end end @@ -16,6 +21,10 @@ module Gitlab @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip end + + def strip_key(key) + key.split(/ /)[0, 2].join(' ') + end end # Init new repository @@ -107,7 +116,7 @@ module Gitlab # def add_key(key_id, key_content) Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, key_content]) + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb new file mode 100644 index 00000000000..d388ab6b879 --- /dev/null +++ b/lib/gitlab/ci/config/node/environment.rb @@ -0,0 +1,68 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an environment. + # + class Environment < Entry + include Validatable + + ALLOWED_KEYS = %i[name url] + + validations do + validate do + unless hash? || string? + errors.add(:config, 'should be a hash or a string') + end + end + + validates :name, presence: true + validates :name, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message } + + validates :name, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + with_options if: :hash? do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :url, + length: { maximum: 255 }, + addressable_url: true, + allow_nil: true + end + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def url + value[:url] + end + + def value + case @config + when String then { name: @config } + when Hash then @config + else {} + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb index 0cbdf7619c0..603334d6793 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/node/job.rb @@ -13,7 +13,7 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :environment, :dependencies + attributes :tags, :allow_failure, :when, :dependencies validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -29,58 +29,53 @@ module Gitlab inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ 'always or manual' } - validates :environment, - type: { - with: String, - message: Gitlab::Regex.environment_name_regex_message } - validates :environment, - format: { - with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } validates :dependencies, array_of_strings: true end end - node :before_script, Script, + node :before_script, Node::Script, description: 'Global before script overridden in this job.' - node :script, Commands, + node :script, Node::Commands, description: 'Commands that will be executed in this job.' - node :stage, Stage, + node :stage, Node::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Stage, + node :type, Node::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Script, + node :after_script, Node::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Cache, + node :cache, Node::Cache, description: 'Cache definition for this job.' - node :image, Image, + node :image, Node::Image, description: 'Image that will be used to execute this job.' - node :services, Services, + node :services, Node::Services, description: 'Services that will be used to execute this job.' - node :only, Trigger, + node :only, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Trigger, + node :except, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Variables, + node :variables, Node::Variables, description: 'Environment variables available for this job.' - node :artifacts, Artifacts, + node :artifacts, Node::Artifacts, description: 'Artifacts configuration for this job.' + node :environment, Node::Environment, + description: 'Environment configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands + :artifacts, :commands, :environment def compose!(deps = nil) super do @@ -133,6 +128,8 @@ module Gitlab only: only, except: except, variables: variables_defined? ? variables : nil, + environment: environment_defined? ? environment : nil, + environment_name: environment_defined? ? environment[:name] : nil, artifacts: artifacts, after_script: after_script } end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index bd681f03173..b164f5a2eea 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,16 +1,16 @@ module Gitlab class ContributionsCalendar - attr_reader :timestamps, :projects, :user + attr_reader :activity_dates, :projects, :user def initialize(projects, user) @projects = projects @user = user end - def timestamps - return @timestamps if @timestamps.present? + def activity_dates + return @activity_dates if @activity_dates.present? - @timestamps = {} + @activity_dates = {} date_from = 1.year.ago events = Event.reorder(nil).contributions.where(author_id: user.id). @@ -19,18 +19,17 @@ module Gitlab select('date(created_at) as date, count(id) as total_amount'). map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a + activity_dates = (1.year.ago.to_date..Date.today).to_a - dates.each do |date| - date_id = date.to_time.to_i.to_s + activity_dates.each do |date| day_events = events.find { |day_events| day_events["date"] == date } if day_events - @timestamps[date_id] = day_events["total_amount"] + @activity_dates[date] = day_events["total_amount"] end end - @timestamps + @activity_dates end def events_by_date(date) diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb new file mode 100644 index 00000000000..b6a89f715fd --- /dev/null +++ b/lib/gitlab/database/date_time.rb @@ -0,0 +1,27 @@ +module Gitlab + module Database + module DateTime + # Find the first of the `end_time_attrs` that isn't `NULL`. Subtract from it + # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval + # along with an alias specified by the `as` parameter. + # + # Note: For MySQL, the interval is returned in seconds. + # For PostgreSQL, the interval is returned as an INTERVAL type. + def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) + diff_fn = if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + + query_so_far.project(diff_fn.as(as)) + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb new file mode 100644 index 00000000000..1444d25ebc7 --- /dev/null +++ b/lib/gitlab/database/median.rb @@ -0,0 +1,112 @@ +# https://www.periscopedata.com/blog/medians-in-sql.html +module Gitlab + module Database + module Median + def median_datetime(arel_table, query_so_far, column_sym) + median_queries = + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym) + elsif Gitlab::Database.mysql? + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + + results = Array.wrap(median_queries).map do |query| + ActiveRecord::Base.connection.execute(query) + end + extract_median(results).presence + end + + def extract_median(results) + result = results.compact.first + + if Gitlab::Database.postgresql? + result = result.first.presence + median = result['median'] if result + median.to_f if median + elsif Gitlab::Database.mysql? + result.to_a.flatten.first + end + end + + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + query = arel_table. + from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). + project(average([arel_table[column_sym]], 'median')). + where( + Arel::Nodes::Between.new( + Arel.sql("(select @row_id := @row_id + 1)"), + Arel::Nodes::And.new( + [Arel.sql('@ct/2.0'), + Arel.sql('@ct/2.0 + 1')] + ) + ) + ). + # Disallow negative values + where(arel_table[column_sym].gteq(0)) + + [ + Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), + Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), + Arel.sql("set @row_id := 0;"), + query.to_sql, + Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") + ] + end + + def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + # Create a CTE with the column we're operating on, row number (after sorting by the column + # we're operating on), and count of the table we're operating on (duplicated across) all rows + # of the CTE. For example, if we're looking to find the median of the `projects.star_count` + # column, the CTE might look like this: + # + # star_count | row_id | ct + # ------------+--------+---- + # 5 | 1 | 3 + # 9 | 2 | 3 + # 15 | 3 | 3 + cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( + cte_table, + arel_table. + project( + arel_table[column_sym].as(column_sym.to_s), + Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), + Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), + arel_table.project("COUNT(1)").as('ct')). + # Disallow negative values + where(arel_table[column_sym].gteq(zero_interval))) + + # From the CTE, select either the middle row or the middle two rows (this is accomplished + # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the + # selected rows, and this is the median value. + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). + where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) + ) + ). + with(query_so_far, cte). + to_sql + end + + private + + def average(args, as) + Arel::Nodes::NamedFunction.new("AVG", args, as) + end + + def extract_epoch(arel_attribute) + Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) + end + + # Need to cast '0' to an INTERVAL before we can check if the interval is positive + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 7584efe4fa8..3ab99360206 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -18,6 +18,14 @@ module Gitlab end end + def committer_hash(email:, name:) + { + email: email, + name: name, + time: Time.now + } + end + def tag_name(ref) ref = ref.to_s if self.tag_ref?(ref) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1882eb8d050..799794c0171 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -5,12 +5,13 @@ module Gitlab DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project, :protocol, :user_access + attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol + @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) end @@ -60,14 +61,26 @@ module Gitlab end def user_download_access_check - unless user_access.can_do_action?(:download_code) + unless user_can_download_code? || build_can_download_code? return build_status_object(false, "You are not allowed to download code from this project.") end build_status_object(true) end + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + def user_push_access_check(changes) + unless authentication_abilities.include?(:push_code) + return build_status_object(false, "You are not allowed to upload code for this project.") + end + if changes.blank? return build_status_object(true) end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index e9725880c5e..605abfabdab 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -3,8 +3,9 @@ module Gitlab class ProjectCreator attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,8 +14,8 @@ module Gitlab def execute project = ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.name, + name: @name, + path: @name, description: repo.description, namespace_id: namespace.id, visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility, diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb562bdcd2c..181e288a014 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.3' + # For every version update, the version history in import_export.md has to be kept up to date. + VERSION = '0.1.4' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index c2e8a1ca5dd..925a952156f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -35,7 +35,9 @@ project_tree: - :deploy_keys - :services - :hooks - - :protected_branches + - protected_branches: + - :merge_access_levels + - :push_access_levels - :labels - milestones: - :events diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b0726268ca6..354ccd64696 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -7,7 +7,9 @@ module Gitlab variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze @@ -17,6 +19,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + FINDER_ATTRIBUTES = %w[title project_id].freeze + def self.create(*args) new(*args).create end @@ -149,7 +153,7 @@ module Gitlab end def parsed_relation_hash - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } end def set_st_diffs @@ -161,14 +165,30 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) - existing_object.assign_attributes(parsed_relation_hash) + events = parsed_relation_hash.delete('events') + + unless events.blank? + existing_object.assign_attributes(events: events) + end + existing_object else relation_class.new(parsed_relation_hash) end end end + + def existing_object + @existing_object ||= + begin + finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES) + existing_object = relation_class.find_or_create_by(finder_hash) + # Done in two steps, as MySQL behaves differently than PostgreSQL using + # the +find_or_create_by+ method and does not return the ID the second time. + existing_object.update(parsed_relation_hash) + existing_object + end + end end end end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index de3fe6d822e..fc08082fc86 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,8 +24,8 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 9100719da87..82cb8cef754 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,7 +70,7 @@ module Gitlab private def user_options(field, value, limit) - options = { attributes: %W(#{config.uid} cn mail dn) } + options = { attributes: user_attributes } options[:size] = limit if limit if field.to_sym == :dn @@ -98,6 +98,10 @@ module Gitlab filter end end + + def user_attributes + %W(#{config.uid} cn mail dn) + end end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb new file mode 100644 index 00000000000..d089a2f9b0b --- /dev/null +++ b/lib/gitlab/lfs_token.rb @@ -0,0 +1,54 @@ +module Gitlab + class LfsToken + attr_accessor :actor + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 1800 + + def initialize(actor) + @actor = + case actor + when DeployKey, User + actor + when Key + actor.user + else + raise 'Bad Actor' + end + end + + def generate + token = Devise.friendly_token(TOKEN_LENGTH) + + Gitlab::Redis.with do |redis| + redis.set(redis_key, token, ex: EXPIRY_TIME) + end + + token + end + + def value + Gitlab::Redis.with do |redis| + redis.get(redis_key) + end + end + + def user? + actor.is_a?(User) + end + + def type + actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + end + + def actor_name + actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + end + + private + + def redis_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ffad5e17c78..776bbcbb5d0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -44,7 +44,7 @@ module Gitlab end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'." end def file_path_regex @@ -52,7 +52,7 @@ module Gitlab end def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." end def directory_traversal_regex @@ -60,7 +60,7 @@ module Gitlab end def directory_traversal_regex_message - "cannot include directory traversal. " + "cannot include directory traversal." end def archive_formats_regex @@ -96,11 +96,11 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end end end |