diff options
Diffstat (limited to 'lib')
45 files changed, 330 insertions, 98 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0950c3d2e88..be659fa4a6a 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -129,12 +129,7 @@ module API end end - # Delete all merged branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/repository/branches/delete_merged + desc 'Delete all merged branches' delete ":id/repository/merged_branches" do DeleteMergedBranchesService.new(user_project, current_user).async_execute diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index f6cb17bafd8..64da7d6b86f 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -38,26 +38,25 @@ module API present key, with: Entities::SSHKey end - # TODO: for 9.0 we should check if params are there with the params block - # grape provides, at this point we'd change behaviour so we can't - # Behaviour now if you don't provide all required params: it renders a - # validation error or two. desc 'Add new deploy key to currently authenticated user' do success Entities::SSHKey end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end post ":id/#{path}" do - attrs = attributes_for_keys [:title, :key] - attrs[:key].strip! if attrs[:key] + params[:key].strip! # Check for an existing key joined to this project - key = user_project.deploy_keys.find_by(key: attrs[:key]) + key = user_project.deploy_keys.find_by(key: params[:key]) if key present key, with: Entities::SSHKey break end # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey @@ -65,7 +64,7 @@ module API end # Create a new deploy key - key = DeployKey.new attrs + key = DeployKey.new(declared_params(include_missing: false)) if key.valid? && user_project.deploy_keys << key present key, with: Entities::SSHKey else diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9f59939e9ae..a07b2a9ca0f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -137,6 +137,7 @@ module API expose :avatar_url expose :web_url expose :request_access_enabled + expose :full_name, :full_path expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do diff --git a/lib/api/services.rb b/lib/api/services.rb index a0abec49438..1456fe4688b 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -661,6 +661,14 @@ module API end trigger_services.each do |service_slug, settings| + helpers do + def chat_command_service(project, service_slug, params) + project.services.active.where(template: false).find do |service| + service.try(:token) == params[:token] && service.to_param == service_slug.underscore + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -679,9 +687,8 @@ module API # This is not accurate, but done to prevent leakage of the project names not_found!('Service') unless project - service = project.find_or_initialize_service(service_slug.underscore) - - result = service.try(:active?) && service.try(:trigger, params) + service = chat_command_service(project, service_slug, params) + result = service.try(:trigger, params) if result status result[:status] || 200 diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 6d04f68c8f9..a3d495a5da0 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -153,7 +153,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object) + data = data_attributes_for(link_content || match, project, object, link: !!link_content) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -172,9 +172,10 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) data_attribute( original: text, + link: link, project: project.id, object_sym => object.id ) diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4d1bc687696..fd6b9704132 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -62,7 +62,7 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) if object.is_a?(ExternalIssue) data_attribute( project: project.id, diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index ab7af1cad21..6640168bfa2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -53,6 +53,10 @@ module Banzai context[:project] end + def skip_project_check? + context[:skip_project_check] + end + def reference_class(type) "gfm gfm-#{type} has-tooltip" end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 026b81ac175..a447e2b8bff 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -20,17 +20,19 @@ module Banzai code = node.text css_classes = "code highlight" lexer = lexer_for(language) + lang = lexer.tag begin code = format(lex(lexer, code)) - css_classes << " js-syntax-highlight #{lexer.tag}" + css_classes << " js-syntax-highlight #{lang}" rescue + lang = nil # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index f842b1fb779..1aa9355b256 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai end def call - return doc if project.nil? + return doc if project.nil? && !skip_project_check? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ @@ -58,7 +58,7 @@ module Banzai # have `gfm` and `gfm-project_member` class names attached for styling. def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| - if username == 'all' + if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) elsif namespace = namespaces[username] link_to_namespace(namespace, link_content: link_content) || match diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index ac7bbcb0d10..b64a1287d4d 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -35,7 +35,8 @@ module Banzai src: element['src'], width: '400', controls: true, - 'data-setup' => '{}') + 'data-setup' => '{}', + 'data-title' => element['title'] || element['alt']) link = doc.document.create_element( 'a', diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5a1f873496c..ac95a79009b 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,6 +1,12 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline + # These filters convert GitLab Flavored Markdown (GFM) to HTML. + # The handlers defined in app/assets/javascripts/copy_as_gfm.js.es6 + # consequently convert that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a handler in + # app/assets/javascripts/copy_as_gfm.js.es6, in reverse order. + # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ Filter::SyntaxHighlightFilter, diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index f31fb6c3f71..74663556cbb 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -52,9 +52,9 @@ module Banzai end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field) + def cacheless_render_field(object, field, options = {}) text = object.__send__(field) - context = object.banzai_render_context(field) + context = object.banzai_render_context(field).merge(options) cacheless_render(text, context) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index c4bdef781f7..8b939663ffd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -18,24 +18,31 @@ module Ci if current_runner.is_runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) return build_not_found! end new_update = current_runner.ensure_runner_queue_value - build = Ci::RegisterBuildService.new.execute(current_runner) + result = Ci::RegisterBuildService.new(current_runner).execute - if build - Gitlab::Metrics.add_event(:build_found, - project: build.project.path_with_namespace) + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) - present build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) + present result.build, with: Entities::BuildDetails + else + Gitlab::Metrics.add_event(:build_not_found) - header 'X-GitLab-Last-Update', new_update + header 'X-GitLab-Last-Update', new_update - build_not_found! + build_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! end end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb index 37e51536e8f..1d7ddeb3e0f 100644 --- a/lib/gitlab/ci/trace_reader.rb +++ b/lib/gitlab/ci/trace_reader.rb @@ -42,6 +42,7 @@ module Gitlab end chunks.join.lines.last(max_lines).join + .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2ff27e46d64..e20f5f6f514 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,7 +9,9 @@ module Gitlab end def ensure_application_settings! - if connect_to_db? + return fake_application_settings unless connect_to_db? + + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available @@ -20,43 +22,23 @@ module Gitlab settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end - settings || fake_application_settings + settings || in_memory_application_settings end def sidekiq_throttling_enabled? current_application_settings.sidekiq_throttling_enabled? end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + fake_application_settings + end + def fake_application_settings - OpenStruct.new( - default_projects_limit: Settings.gitlab['default_projects_limit'], - default_branch_protection: Settings.gitlab['default_branch_protection'], - signup_enabled: Settings.gitlab['signup_enabled'], - signin_enabled: Settings.gitlab['signin_enabled'], - gravatar_enabled: Settings.gravatar['enabled'], - koding_enabled: false, - plantuml_enabled: false, - sign_in_text: nil, - after_sign_up_text: nil, - help_page_text: nil, - shared_runners_text: nil, - restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - max_attachment_size: Settings.gitlab['max_attachment_size'], - session_expire_delay: Settings.gitlab['session_expire_delay'], - default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], - default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], - shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Settings.artifacts['max_size'], - require_two_factor_authentication: false, - two_factor_grace_period: 48, - akismet_enabled: false, - repository_checks_enabled: true, - container_registry_token_expire_delay: 5, - user_default_external: false, - sidekiq_throttling_enabled: false, - ) + OpenStruct.new(::ApplicationSetting.defaults) end private diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 74bbcdcb3dd..559e3939da6 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -13,7 +13,7 @@ module Gitlab end def as_json - AnalyticsStageSerializer.new.represent(self).as_json + AnalyticsStageSerializer.new.represent(self) end def title diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 5245b9ca8fc..d5bf6149749 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 0d8da99455e..3df9cbdcfce 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -16,7 +16,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + AnalyticsIssueSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 88a8710dbe6..7d342a2d2cb 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -37,7 +37,7 @@ module Gitlab def serialize_commit(event, st_commit, query) commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) - AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit).as_json + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end end end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index 4df0bd06393..4c7b3f4467f 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -15,7 +15,7 @@ module Gitlab end def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index b34baf5b081..fc77bd86097 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -16,7 +16,7 @@ module Gitlab private def serialize(summary_object) - AnalyticsSummarySerializer.new.represent(summary_object).as_json + AnalyticsSummarySerializer.new.represent(summary_object) end end end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index a34731a5fcd..36c0260dbfe 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -23,7 +23,7 @@ module Gitlab private def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']).as_json + AnalyticsBuildSerializer.new.represent(event['build']) end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 3f635be22ba..a55adc9b1c8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class ProjectCreator + include Gitlab::CurrentSettings + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -34,7 +36,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end # diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index eb667a85b78..d679edec36b 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.5' + VERSION = '0.1.6' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 08ad3274b38..416194e57d7 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,7 +39,6 @@ project_tree: - :author - :events - :statuses - - :variables - :triggers - :deploy_keys - :services diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e43cce768..0319d7707a8 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -4,7 +4,6 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', - variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', hooks: 'ProjectHook', @@ -24,6 +23,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + def self.create(*args) new(*args).create end @@ -61,7 +62,9 @@ module Gitlab update_project_references handle_group_label if group_label? - reset_ci_tokens if @relation_name == 'Ci::Trigger' + reset_tokens! + remove_encrypted_attributes! + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff end @@ -140,11 +143,22 @@ module Gitlab end end - def reset_ci_tokens - return unless Gitlab::ImportExport.reset_tokens? + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) # If we import/export a project to the same instance, tokens will have to be reset. - @relation_hash['token'] = nil + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end end def relation_class diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 45958710c13..52276cbcd9a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -5,8 +5,6 @@ # module Gitlab module ImportSources - extend CurrentSettings - ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb new file mode 100644 index 00000000000..8db91d25a4b --- /dev/null +++ b/lib/gitlab/job_waiter.rb @@ -0,0 +1,27 @@ +module Gitlab + # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + class JobWaiter + # The sleep interval between checking keys, in seconds. + INTERVAL = 0.1 + + # jobs - The job IDs to wait for. + def initialize(jobs) + @jobs = jobs + end + + # Waits for all the jobs to be completed. + # + # timeout - The maximum amount of seconds to block the caller for. This + # ensures we don't indefinitely block a caller in case a job takes + # long to process, or is never processed. + def wait(timeout = 60) + start = Time.current + + while (Time.current - start) <= timeout + break if SidekiqStatus.all_completed?(@jobs) + + sleep(INTERVAL) # to not overload Redis too much. + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6bdf3db9cb8..db325c00705 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -71,6 +71,14 @@ module Gitlab ) end + def single_commit_result? + commits_count == 1 && total_result_count == 1 + end + + def total_result_count + issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + end + private def blobs @@ -114,7 +122,25 @@ module Gitlab end def commits - @commits ||= project.repository.find_commits_by_message(query) + @commits ||= find_commits(query) + end + + def find_commits(query) + return [] unless Ability.allowed?(@current_user, :download_code, @project) + + commits = find_commits_by_message(query) + commit_by_sha = find_commit_by_sha(query) + commits |= [commit_by_sha] if commit_by_sha + commits + end + + def find_commits_by_message(query) + project.repository.find_commits_by_message(query) + end + + def find_commit_by_sha(query) + key = query.strip + project.repository.commit(key) if Commit.valid_hash?(key) end def project_ids_relation diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 35212992698..c9c65f76f4b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,10 @@ module Gitlab @milestones_count ||= milestones.count end + def single_commit_result? + false + end + private def projects diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/shell.rb index 82e194c1af1..82e194c1af1 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/shell.rb diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..fbe2a7a0d72 100644 --- a/lib/gitlab/backend/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb new file mode 100644 index 00000000000..aadc401ff8d --- /dev/null +++ b/lib/gitlab/sidekiq_status.rb @@ -0,0 +1,66 @@ +module Gitlab + # The SidekiqStatus module and its child classes can be used for checking if a + # Sidekiq job has been processed or not. + # + # To check if a job has been completed, simply pass the job ID to the + # `completed?` method: + # + # job_id = SomeWorker.perform_async(...) + # + # if Gitlab::SidekiqStatus.completed?(job_id) + # ... + # end + # + # For each job ID registered a separate key is stored in Redis, making lookups + # much faster than using Sidekiq's built-in job finding/status API. These keys + # expire after a certain period of time to prevent storing too many keys in + # Redis. + module SidekiqStatus + STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + + # The default time (in seconds) after which a status key is expired + # automatically. The default of 30 minutes should be more than sufficient + # for most jobs. + DEFAULT_EXPIRATION = 30.minutes.to_i + + # Starts tracking of the given job. + # + # jid - The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.set(jid, expire = DEFAULT_EXPIRATION) + Sidekiq.redis do |redis| + redis.set(key_for(jid), 1, ex: expire) + end + end + + # Stops the tracking of the given job. + # + # jid - The Sidekiq job ID to remove. + def self.unset(jid) + Sidekiq.redis do |redis| + redis.del(key_for(jid)) + end + end + + # Returns true if all the given job have been completed. + # + # jids - The Sidekiq job IDs to check. + # + # Returns true or false. + def self.all_completed?(jids) + keys = jids.map { |jid| key_for(jid) } + + responses = Sidekiq.redis do |redis| + redis.pipelined do + keys.each { |key| redis.exists(key) } + end + end + + responses.all? { |value| !value } + end + + def self.key_for(jid) + STATUS_KEY % jid + end + end +end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb new file mode 100644 index 00000000000..779a9998b22 --- /dev/null +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -0,0 +1,10 @@ +module Gitlab + module SidekiqStatus + class ClientMiddleware + def call(_, job, _, _) + SidekiqStatus.set(job['jid']) + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb new file mode 100644 index 00000000000..31dfa46ff9d --- /dev/null +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqStatus + class ServerMiddleware + def call(worker, job, queue) + ret = yield + + SidekiqStatus.unset(job['jid']) + + ret + end + end + end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index f3567f3ef85..e78d0c34a02 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -61,7 +61,7 @@ module Gitlab "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), "Install gems" => %W(bundle), "Migrate DB" => %W(bundle exec rake db:migrate), - "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), + "Recompile assets" => %W(bundle exec rake gitlab:assets:clean gitlab:assets:compile), "Clear cache" => %W(bundle exec rake cache:clear) } end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 83c8ba5c1cf..dbfe0941e4d 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -1,6 +1,8 @@ module Gitlab module View module Presenter + CannotOverrideMethodError = Class.new(StandardError) + module Base extend ActiveSupport::Concern diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index f4d330c590e..387ff0f5d43 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -8,6 +8,10 @@ module Gitlab @subject = subject attributes.each do |key, value| + if subject.respond_to?(key) + raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + end + define_singleton_method(key) { value } end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..c7953af29dd 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,6 +11,7 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } end diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index ec2903b7ec6..e55c0d6ac49 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -8,21 +8,31 @@ module Mattermost @user = user end - private - def with_session(&blk) Mattermost::Session.new(user).with_session(&blk) end - def json_get(path, options = {}) + private + + # Should be used in a session manually + def get(session, path, options = {}) + json_response session.get(path, options) + end + + # Should be used in a session manually + def post(session, path, options = {}) + json_response session.post(path, options) + end + + def session_get(path, options = {}) with_session do |session| - json_response session.get(path, options) + get(session, path, options) end end - def json_post(path, options = {}) + def session_post(path, options = {}) with_session do |session| - json_response session.post(path, options) + post(session, path, options) end end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb index d1e4bb0eccf..33e450d7f0a 100644 --- a/lib/mattermost/command.rb +++ b/lib/mattermost/command.rb @@ -1,7 +1,7 @@ module Mattermost class Command < Client def create(params) - response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create", body: params.to_json) response['token'] diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 784eca6ab5a..09dfd082b3a 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,7 +1,7 @@ module Mattermost class Team < Client def all - json_get('/api/v3/teams/all') + session_get('/api/v3/teams/all') end end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index adea4c7a747..ab46c47d8f5 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -31,8 +31,8 @@ echo 'Deploy: Bundle and migrate' sudo -u git -H bundle --without aws development test mysql --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production -sudo -u git -H bundle exec rake assets:clean RAILS_ENV=production -sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:compile RAILS_ENV=production sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production # return stashed changes (if necessary) diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake new file mode 100644 index 00000000000..5d884bf9f66 --- /dev/null +++ b/lib/tasks/gitlab/assets.rake @@ -0,0 +1,47 @@ +namespace :gitlab do + namespace :assets do + desc 'GitLab | Assets | Compile all frontend assets' + task :compile do + Rake::Task['assets:precompile'].invoke + Rake::Task['gitlab:assets:fix_urls'].invoke + end + + desc 'GitLab | Assets | Clean up old compiled frontend assets' + task :clean do + Rake::Task['assets:clean'].invoke + end + + desc 'GitLab | Assets | Remove all compiled frontend assets' + task :purge do + Rake::Task['assets:clobber'].invoke + end + + desc 'GitLab | Assets | Fix all absolute url references in CSS' + task :fix_urls do + css_files = Dir['public/assets/*.css'] + css_files.each do | file | + # replace url(/assets/*) with url(./*) + puts "Fixing #{file}" + system "sed", "-i", "-e", 's/url(\([\"\']\?\)\/assets\//url(\1.\//g', file + + # rewrite the corresponding gzip file (if it exists) + gzip = "#{file}.gz" + if File.exist?(gzip) + puts "Fixing #{gzip}" + + FileUtils.rm(gzip) + mtime = File.stat(file).mtime + + File.open(gzip, 'wb+') do |f| + gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) + gz.mtime = mtime + gz.write IO.binread(file) + gz.close + + File.utime(mtime, mtime, f.path) + end + end + end + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index dffea8ed155..f7c831892ee 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -11,8 +11,10 @@ namespace :gitlab do gem_version = run_command(%W(gem --version)) # check Bundler version bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) - # check Bundler version + # check Rake version rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) + # check redis version + redis_version = run_and_match(%W(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a puts "" puts "System information".color(:yellow) @@ -24,6 +26,7 @@ namespace :gitlab do puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" + puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" |