diff options
author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-04-22 21:01:50 +0200 |
---|---|---|
committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-04-22 21:01:50 +0200 |
commit | 8c8f6db4578051300794c7032bc63d68f70cce16 (patch) | |
tree | f48f82ed1418ff9a6d810b4f354629d7a275b606 /lib | |
parent | a8231ea1befd803fb5892ea3e6679219f5d7d8e5 (diff) | |
parent | 1005389f70070245092c1ae5f3f9b10b8e7c102e (diff) | |
download | gitlab-ce-8c8f6db4578051300794c7032bc63d68f70cce16.tar.gz |
Merge branch 'master' into feature/gb/manual-actions-protected-branches-permissions
* master: (274 commits)
Update VERSION to 9.2.0-pre
Update CHANGELOG.md for 9.1.0
Update Auto Deploy documentation
Disable import URL field in New project form since it's hidden by default
Remove reference to burndown charts since they don't exist for ce.
Use master_password for Sentinel
Refactor Discussions docs
Start versioning cached markdown fields
Refactor add_users method for project and group
Improved the spec Now correctly tests against different forms
Refactor environments components into vue files - part 3
Adding animation for all dropdown
fix placeholder visibility
submodule_links: handle urls that don't end with .git
Add help regarding vue resource and where to include it
Append .json to ajax endpoint to prevent browser to display raw json
Fixed the preview keyboard shortcut focusing wrong tab
Fix broken link
Added new discussions docs
Started on resolvable discussions docs
...
Diffstat (limited to 'lib')
36 files changed, 299 insertions, 160 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 66b37fd2bcc..621b9dcecd9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,7 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -140,7 +140,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9919762cd82..64ab6f01eb5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -18,6 +18,12 @@ module API expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +31,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity diff --git a/lib/api/files.rb b/lib/api/files.rb index 33fc970dc09..e6ea12c5ab7 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,7 +5,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -130,7 +130,7 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] != :success render_api_error!(result[:message], 400) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 810e5063996..718f936a1fc 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,6 +60,12 @@ module API rescue JSON::ParserError {} end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 215bc03d0e9..5b48ee8665f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -40,6 +40,8 @@ module API response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 05423c17449..244725bb292 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -35,7 +35,7 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 50842370947..db4b31b55bc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,7 +11,7 @@ module API optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -103,6 +103,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -205,7 +206,7 @@ module API # CE at_least_one_of_ce = [ - :builds_enabled, + :jobs_enabled, :container_registry_enabled, :default_branch, :description, @@ -236,6 +237,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success diff --git a/lib/api/users.rb b/lib/api/users.rb index eedc59f8636..46f221f68fe 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -39,10 +39,13 @@ module API params do # CE optional :username, type: String, desc: 'Get a single user with a specific username' + optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + all_or_none_of :extern_uid, :provider use :pagination end @@ -51,14 +54,17 @@ module API render_api_error!("Not authorized.", 403) end - if params[:username].present? - users = User.where(username: params[:username]) - else - users = User.all - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - users = users.external if params[:external] && current_user.admin? + authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + + users = User.all + users = User.where(username: params[:username]) if params[:username] + users = users.active if params[:active] + users = users.search(params[:search]) if params[:search].present? + users = users.blocked if params[:blocked] + + if current_user.admin? + users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider] + users = users.external if params[:external] end entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic @@ -534,6 +540,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 3414a2883e5..674de592f0a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -53,7 +53,7 @@ module API attrs = declared_params.dup branch = attrs.delete(:branch_name) - attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + attrs.merge!(start_branch: branch, branch_name: branch) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -131,7 +131,7 @@ module API commit_params = { commit: commit, start_branch: params[:branch], - target_branch: params[:branch] + branch_name: params[:branch] } result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb index 13542b0c71c..c76acc86504 100644 --- a/lib/api/v3/files.rb +++ b/lib/api/v3/files.rb @@ -6,7 +6,7 @@ module API { file_path: attrs[:file_path], start_branch: attrs[:branch], - target_branch: attrs[:branch], + branch_name: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -123,7 +123,7 @@ module API file_params = declared_params(include_missing: false) file_params[:branch] = file_params.delete(:branch_name) - result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 6b78aa795b4..327ea9449a1 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -9,12 +9,14 @@ module Banzai VISIBLE_STATES = %w(closed merged).freeze def call + return doc unless context[:issuable_state_filter_enabled] + extractor = Banzai::IssuableExtractor.new(project, current_user) issuables = extractor.extract([doc]) issuables.each do |node, issuable| - if VISIBLE_STATES.include?(issuable.state) - node.children.last.content += " [#{issuable.state}]" + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index dabf71d6aeb..c2503fa2adc 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -136,7 +136,8 @@ module Banzai nodes.each_with_object({}) do |node, hash| if node.has_attribute?(attribute) - hash[node] = objects_by_id[node.attr(attribute).to_i] + obj = objects_by_id[node.attr(attribute).to_i] + hash[node] = obj if obj end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 1020452480a..b439b0ee29b 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -172,7 +172,7 @@ module Ci close_open_tags() OpenStruct.new( - html: @out, + html: @out.force_encoding(Encoding.default_external), state: state, append: append, truncated: truncated, diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index a4b5f2aba6c..61849a40383 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -15,7 +15,7 @@ module ContainerRegistry LEVELS_SUPPORTED = 3 def initialize(path) - @path = path + @path = path.to_s.downcase end def valid? @@ -25,7 +25,7 @@ module ContainerRegistry end def components - @components ||= @path.to_s.split('/') + @components ||= @path.split('/') end def nodes @@ -48,7 +48,7 @@ module ContainerRegistry end def root_repository? - @path == repository_project.full_path + @path == project_path end def repository_project @@ -60,7 +60,13 @@ module ContainerRegistry def repository_name return unless has_project? - @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^#{Regexp.escape(project_path)}/?)) + end + + def project_path + return unless has_project? + + repository_project.full_path.downcase end def to_s diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 41dcf846fed..fa462cbe095 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -4,7 +4,7 @@ module Gitlab # This was inspired from: http://stackoverflow.com/a/10219411/1520132 class Stream BUFFER_SIZE = 4096 - LIMIT_SIZE = 50.kilobytes + LIMIT_SIZE = 500.kilobytes attr_reader :stream @@ -14,6 +14,7 @@ module Gitlab def initialize @stream = yield + @stream&.binmode end def valid? @@ -25,11 +26,10 @@ module Gitlab end def limit(last_bytes = LIMIT_SIZE) - stream_size = size - if stream_size < last_bytes - last_bytes = stream_size + if last_bytes < size + stream.seek(-last_bytes, IO::SEEK_END) + stream.readline end - stream.seek(-last_bytes, IO::SEEK_END) end def append(data, offset) @@ -52,7 +52,7 @@ module Gitlab read_last_lines(last_lines) else stream.read - end + end.force_encoding(Encoding.default_external) end def html_with_state(state = nil) @@ -61,8 +61,8 @@ module Gitlab def html(last_lines: nil) text = raw(last_lines: last_lines) - stream = StringIO.new(text) - ::Ci::Ansi2html.convert(stream).html + buffer = StringIO.new(text) + ::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) @@ -114,7 +114,6 @@ module Gitlab end chunks.join.lines.last(last_lines).join - .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 0e22f2189ee..c66b0435f3a 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -7,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -32,10 +34,6 @@ module Gitlab sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..df491f060bf 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..bb4fdd1f1f4 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -32,6 +32,10 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, + project: handler.try(:project), + handler: handler.class.name) + handler.execute end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index e56eb0d3beb..98fd4e78126 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -8,7 +8,7 @@ module Gitlab # the user. We load as much as we can for encoding detection # (Linguist) and LFS pointer parsing. All other cases where we need full # blob data should use load_all_data!. - MAX_DATA_DISPLAY_SIZE = 10485760 + MAX_DATA_DISPLAY_SIZE = 10.megabytes attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary @@ -153,7 +153,7 @@ module Gitlab def lfs_size if has_lfs_version_key? size = data.match(/(?<=size )([0-9]+)/) - return size[1] if size + return size[1].to_i if size end nil diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index af1744c9c46..1add037fa5f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,8 +1,12 @@ module Gitlab module Git class Index + IndexError = Class.new(StandardError) + DEFAULT_MODE = 0o100644 + ACTIONS = %w(create create_dir update move delete).freeze + attr_reader :repository, :raw_index def initialize(repository) @@ -23,9 +27,8 @@ module Gitlab def create(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end add_blob(options) @@ -34,13 +37,12 @@ module Gitlab def create_dir(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end if dir_exists?(options[:file_path]) - raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + raise IndexError, "A directory with this name already exists" end options = options.dup @@ -55,7 +57,7 @@ module Gitlab file_entry = get(options[:file_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" end add_blob(options, mode: file_entry[:mode]) @@ -66,7 +68,11 @@ module Gitlab file_entry = get(options[:previous_path]) unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + raise IndexError, "A file with this name doesn't exist" + end + + if get(options[:file_path]) + raise IndexError, "A file with this name already exists" end raw_index.remove(options[:previous_path]) @@ -77,9 +83,8 @@ module Gitlab def delete(options) options = normalize_options(options) - file_entry = get(options[:file_path]) - unless file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + unless get(options[:file_path]) + raise IndexError, "A file with this name doesn't exist" end raw_index.remove(options[:file_path]) @@ -95,10 +100,20 @@ module Gitlab end def normalize_path(path) + unless path + raise IndexError, "You must provide a file path" + end + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - if pathname.each_filename.include?('..') - raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + pathname.each_filename do |segment| + if segment == '..' + raise IndexError, 'Path cannot include directory traversal' + end + + unless segment =~ Gitlab::Regex.file_name_regex + raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" + end end pathname.to_s @@ -106,6 +121,10 @@ module Gitlab def add_blob(options, mode: nil) content = options[:content] + unless content + raise IndexError, "You must provide content" + end + content = Base64.decode64(content) if options[:encoding] == 'base64' detect = CharlockHolmes::EncodingDetector.new.detect(content) @@ -119,7 +138,7 @@ module Gitlab raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) rescue Rugged::IndexError => e - raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + raise IndexError, e.message end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index bcdf1b1faa8..c69676a1dac 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -15,7 +15,7 @@ module Gitlab end unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect}" + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" end @addresses[name] = address diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index dda371e6554..49285e35251 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,6 +1,11 @@ module Gitlab module MarkupHelper - module_function + extend self + + MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze + ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze + OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -8,10 +13,7 @@ module Gitlab # # Returns boolean def markup?(filename) - gitlab_markdown?(filename) || - asciidoc?(filename) || - filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki - .mediawiki .rst)) + EXTENSIONS.include?(extension(filename)) end # Public: Determines if a given filename is compatible with @@ -21,7 +23,7 @@ module Gitlab # # Returns boolean def gitlab_markdown?(filename) - filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown)) + MARKDOWN_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename has AsciiDoc extension. @@ -30,7 +32,7 @@ module Gitlab # # Returns boolean def asciidoc?(filename) - filename.downcase.end_with?(*%w(.adoc .ad .asciidoc)) + ASCIIDOC_EXTENSIONS.include?(extension(filename)) end # Public: Determines if the given filename is plain text. @@ -39,12 +41,17 @@ module Gitlab # # Returns boolean def plain?(filename) - filename.downcase.end_with?('.txt') || - filename.casecmp('readme').zero? + extension(filename) == 'txt' || filename.casecmp('readme').zero? end def previewable?(filename) markup?(filename) end + + private + + def extension(filename) + File.extname(filename).downcase.delete('.') + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..c6dfa4ad9bd 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -138,6 +138,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +154,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f98481c6d3a..6e42d8941fb 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::CreateService.new(nil, user_params).build + Users::BuildService.new(nil, user_params).execute end def user_attributes diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e599dd4a656..08b061d5e31 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -73,22 +73,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def file_path_regex - @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze - end - - def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." - end - - def directory_traversal_regex - @directory_traversal_regex ||= /\.{2}/.freeze - end - - def directory_traversal_regex_message - "cannot include directory traversal." - end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..6aca6db3123 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,65 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + services: Service.where(active: true).count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 9f6cfe3957c..8079c6e416c 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -7,10 +7,10 @@ namespace :gitlab do abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end - tag = "v#{Gitlab::GitalyClient.expected_server_version}" + version = Gitlab::GitalyClient.expected_server_version repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index dd2fda54e62..95687066819 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,19 +1,18 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:tag, :repo] => :environment do |t, args| + task :install, [:repo] => :environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = "v#{default_version}" - args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index cdba2262bc2..e3c9d3b491c 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -147,41 +147,30 @@ module Gitlab Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end - def checkout_or_clone_tag(tag:, repo:, target_dir:) - if Dir.exist?(target_dir) - checkout_tag(tag, target_dir) - else - clone_repo(repo, target_dir) - end + def checkout_or_clone_version(version:, repo:, target_dir:) + version = + if version.starts_with?("=") + version.sub(/\A=/, '') # tag or branch + else + "v#{version}" # tag + end - reset_to_tag(tag, target_dir) + clone_repo(repo, target_dir) unless Dir.exist?(target_dir) + checkout_version(version, target_dir) + reset_to_version(version, target_dir) end def clone_repo(repo, target_dir) run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) end - def checkout_tag(tag, target_dir) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + def checkout_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}]) end - def reset_to_tag(tag_wanted, target_dir) - tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end - - if tag - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) - else - raise Gitlab::TaskFailedError - end + def reset_to_version(version, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}]) end end end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index baea94bf8ca..a00b02188cf 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -7,10 +7,10 @@ namespace :gitlab do abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end - tag = "v#{Gitlab::Workhorse.version}" + version = Gitlab::Workhorse.version repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 15131fbf755..a9dad6a1bf0 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -52,7 +52,6 @@ class NewImporter < ::Gitlab::GithubImport::Importer project.repository.add_remote(project.import_type, project.import_url) project.repository.set_remote_as_mirror(project.import_type) project.repository.fetch_remote(project.import_type, forced: true) - project.repository.remove_remote(project.import_type) rescue => e # Expire cache to prevent scenarios such as: # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true |