Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

13 files changed, 50 insertions, 184 deletions

diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 0a167104bf4..c5498d0da1a 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,4 +1,3 @@
-require_dependency 'settings'
 require_dependency 'gitlab/popen'
 
 module Gitlab
@@ -30,6 +29,6 @@ module Gitlab
   end
 
   def self.dev_env_or_com?
-    Rails.env.test? || Rails.env.development? || org? || com?
+    Rails.env.development? || org? || com?
   end
 end
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 1a25138e7d6..4ca5a78e068 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -75,10 +75,11 @@ module Gitlab
       end
 
       def mv_repo(project)
-        FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
+        storage_path = storage_path_for_shard(project.repository_storage)
+        FileUtils.mv(repo_path, project.repository.path_to_repo)
 
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
         end
 
         true
@@ -88,6 +89,10 @@ module Gitlab
         false
       end
 
+      def storage_path_for_shard(shard)
+        Gitlab.config.repositories.storages[shard].legacy_disk_path
+      end
+
       def find_or_create_groups
         return nil unless group_path.present?
 
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 963f0228c1a..dc005a2e508 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -54,14 +54,14 @@ module Gitlab
       end
 
       def exist?
-        trace_artifact&.exists? || job.chunks.any? || current_path.present? || old_trace.present?
+        trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
           if trace_artifact
             trace_artifact.open
-          elsif job.chunks.any?
+          elsif job.trace_chunks.any?
             Gitlab::Ci::Trace::ChunkedIO.new(job)
           elsif current_path
             File.open(current_path, "rb")
@@ -100,7 +100,7 @@ module Gitlab
           FileUtils.rm(trace_path, force: true)
         end
 
-        job.chunks.fast_destroy_all
+        job.trace_chunks.fast_destroy_all
         job.erase_old_trace!
       end
 
@@ -108,7 +108,7 @@ module Gitlab
         raise ArchiveError, 'Already archived' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        if job.chunks.any?
+        if job.trace_chunks.any?
           Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
             archive_stream!(stream)
             stream.delete!
@@ -130,7 +130,7 @@ module Gitlab
 
       def archive_stream!(stream)
         clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path|
-          create_job_trace!(job, clone_path)
+          create_build_trace!(job, clone_path)
         end
       end
 
@@ -146,7 +146,7 @@ module Gitlab
         end
       end
 
-      def create_job_trace!(job, path)
+      def create_build_trace!(job, path)
         File.open(path) do |stream|
           job.create_job_artifacts_trace!(
             project: job.project,
diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb
index 9d4243d152a..1bb8420a220 100644
--- a/lib/gitlab/ci/trace/chunked_io.rb
+++ b/lib/gitlab/ci/trace/chunked_io.rb
@@ -5,18 +5,18 @@ module Gitlab
   module Ci
     class Trace
       class ChunkedIO
-        CHUNK_SIZE = ::Ci::JobTraceChunk::CHUNK_SIZE
+        CHUNK_SIZE = ::Ci::BuildTraceChunk::CHUNK_SIZE
 
         FailedToGetChunkError = Class.new(StandardError)
 
-        attr_reader :job
+        attr_reader :build
         attr_reader :tell, :size
         attr_reader :chunk, :chunk_range
 
         alias_method :pos, :tell
 
-        def initialize(job, &block)
-          @job = job
+        def initialize(build, &block)
+          @build = build
           @chunks_cache = []
           @tell = 0
           @size = calculate_size
@@ -140,10 +140,10 @@ module Gitlab
           @size = offset
 
           # remove all next chunks
-          job_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
+          trace_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
 
           # truncate current chunk
-          current_chunk.truncate(chunk_offset) if chunk_offset != 0
+          current_chunk.truncate(chunk_offset)
         ensure
           invalidate_chunk_cache
         end
@@ -157,7 +157,7 @@ module Gitlab
         end
 
         def destroy!
-          job_chunks.fast_destroy_all
+          trace_chunks.fast_destroy_all
           @tell = @size = 0
         ensure
           invalidate_chunk_cache
@@ -206,23 +206,23 @@ module Gitlab
         end
 
         def current_chunk
-          @chunks_cache[chunk_index] ||= job_chunks.find_by(chunk_index: chunk_index)
+          @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index)
         end
 
         def build_chunk
-          @chunks_cache[chunk_index] = ::Ci::JobTraceChunk.new(job: job, chunk_index: chunk_index)
+          @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index)
         end
 
         def ensure_chunk
           current_chunk || build_chunk
         end
 
-        def job_chunks
-          ::Ci::JobTraceChunk.where(job: job)
+        def trace_chunks
+          ::Ci::BuildTraceChunk.where(build: build)
         end
 
         def calculate_size
-          job_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
+          trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i
         end
       end
     end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index a71040e5e56..89e36ecbb15 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -52,7 +52,6 @@ module Gitlab
 
           stream.seek(0, IO::SEEK_SET)
           stream.write(data)
-          stream.truncate(data.bytesize)
           stream.flush()
         end
 
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index 05b86f32ce2..73971af6a74 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -62,21 +62,20 @@ module Gitlab
           end
 
           def move_repositories(namespace, old_full_path, new_full_path)
-            repo_paths_for_namespace(namespace).each do |repository_storage_path|
+            repo_shards_for_namespace(namespace).each do |repository_storage|
               # Ensure old directory exists before moving it
-              gitlab_shell.add_namespace(repository_storage_path, old_full_path)
+              gitlab_shell.add_namespace(repository_storage, old_full_path)
 
-              unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-                message = "Exception moving path #{repository_storage_path} \
-                           from #{old_full_path} to #{new_full_path}"
+              unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
+                message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
                 Rails.logger.error message
               end
             end
           end
 
-          def repo_paths_for_namespace(namespace)
+          def repo_shards_for_namespace(namespace)
             projects_for_namespace(namespace).distinct.select(:repository_storage)
-              .map(&:repository_storage_path)
+              .map(&:repository_storage)
           end
 
           def projects_for_namespace(namespace)
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 979225dd216..827aeb12a02 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -51,7 +51,7 @@ module Gitlab
           end
 
           def move_repository(project, old_path, new_path)
-            unless gitlab_shell.mv_repository(project.repository_storage_path,
+            unless gitlab_shell.mv_repository(project.repository_storage,
                                               old_path,
                                               new_path)
               Rails.logger.error "Error moving #{old_path} to #{new_path}"
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index af203ff711d..b713fa7e1cd 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.2'.freeze
+    VERSION = '0.2.3'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index ac4ac537a8a..156115f8a8f 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -65,11 +65,11 @@ module Gitlab
 
     # Init new repository
     #
-    # storage - project's storage name
+    # storage - the shard key
     # name - project disk path
     #
     # Ex.
-    #   create_repository("/path/to/storage", "gitlab/gitlab-ci")
+    #   create_repository("default", "gitlab/gitlab-ci")
     #
     def create_repository(storage, name)
       relative_path = name.dup
@@ -291,13 +291,13 @@ module Gitlab
     # Add empty directory for storing repositories
     #
     # Ex.
-    #   add_namespace("/path/to/storage", "gitlab")
+    #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:add_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).add(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
         else
           path = full_path(storage, name)
           FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -313,13 +313,13 @@ module Gitlab
     # Every repository inside this directory will be removed too
     #
     # Ex.
-    #   rm_namespace("/path/to/storage", "gitlab")
+    #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:remove_namespace,
                                status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).remove(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
         else
           FileUtils.rm_r(full_path(storage, name), force: true)
         end
@@ -338,7 +338,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:rename_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).rename(old_name, new_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .rename(old_name, new_name)
         else
           break false if exists?(storage, new_name) || !exists?(storage, old_name)
 
@@ -374,7 +375,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:namespace_exists,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).exists?(dir_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .exists?(dir_name)
         else
           File.exist?(full_path(storage, dir_name))
         end
@@ -398,7 +400,7 @@ module Gitlab
     def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
 
-      File.join(storage, dir_name)
+      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
     end
 
     def gitlab_shell_projects_path
@@ -475,14 +477,6 @@ module Gitlab
       Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
     end
 
-    def gitaly_namespace_client(storage_path)
-      storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
-        value.legacy_disk_path == storage_path
-      end
-
-      Gitlab::GitalyClient::NamespaceService.new(storage)
-    end
-
     def git_timeout
       Gitlab.config.gitlab_shell.git_timeout
     end
diff --git a/lib/settings.rb b/lib/settings.rb
deleted file mode 100644
index 69d637761ea..00000000000
--- a/lib/settings.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-require 'settingslogic'
-
-class Settings < Settingslogic
-  source ENV.fetch('GITLAB_CONFIG') { Pathname.new(File.expand_path('..', __dir__)).join('config/gitlab.yml') }
-  namespace ENV.fetch('GITLAB_ENV') { Rails.env }
-
-  class << self
-    def gitlab_on_standard_port?
-      on_standard_port?(gitlab)
-    end
-
-    def host_without_www(url)
-      host(url).sub('www.', '')
-    end
-
-    def build_gitlab_ci_url
-      custom_port =
-        if on_standard_port?(gitlab)
-          nil
-        else
-          ":#{gitlab.port}"
-        end
-
-      [
-        gitlab.protocol,
-        "://",
-        gitlab.host,
-        custom_port,
-        gitlab.relative_url_root
-      ].join('')
-    end
-
-    def build_pages_url
-      base_url(pages).join('')
-    end
-
-    def build_gitlab_shell_ssh_path_prefix
-      user_host = "#{gitlab_shell.ssh_user}@#{gitlab_shell.ssh_host}"
-
-      if gitlab_shell.ssh_port != 22
-        "ssh://#{user_host}:#{gitlab_shell.ssh_port}/"
-      else
-        if gitlab_shell.ssh_host.include? ':'
-          "[#{user_host}]:"
-        else
-          "#{user_host}:"
-        end
-      end
-    end
-
-    def build_base_gitlab_url
-      base_url(gitlab).join('')
-    end
-
-    def build_gitlab_url
-      (base_url(gitlab) + [gitlab.relative_url_root]).join('')
-    end
-
-    # check that values in `current` (string or integer) is a contant in `modul`.
-    def verify_constant_array(modul, current, default)
-      values = default || []
-      unless current.nil?
-        values = []
-        current.each do |constant|
-          values.push(verify_constant(modul, constant, nil))
-        end
-        values.delete_if { |value| value.nil? }
-      end
-
-      values
-    end
-
-    # check that `current` (string or integer) is a contant in `modul`.
-    def verify_constant(modul, current, default)
-      constant = modul.constants.find { |name| modul.const_get(name) == current }
-      value = constant.nil? ? default : modul.const_get(constant)
-      if current.is_a? String
-        value = modul.const_get(current.upcase) rescue default
-      end
-
-      value
-    end
-
-    def absolute(path)
-      File.expand_path(path, Rails.root)
-    end
-
-    private
-
-    def base_url(config)
-      custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
-
-      [
-        config.protocol,
-        "://",
-        config.host,
-        custom_port
-      ]
-    end
-
-    def on_standard_port?(config)
-      config.port.to_i == (config.https ? 443 : 80)
-    end
-
-    # Extract the host part of the given +url+.
-    def host(url)
-      url = url.downcase
-      url = "http://#{url}" unless url.start_with?('http')
-
-      # Get rid of the path so that we don't even have to encode it
-      url_without_path = url.sub(%r{(https?://[^/]+)/?.*}, '\1')
-
-      URI.parse(url_without_path).host
-    end
-
-    # Runs every minute in a random ten-minute period on Sundays, to balance the
-    # load on the server receiving these pings. The usage ping is safe to run
-    # multiple times because of a 24 hour exclusive lock.
-    def cron_for_usage_ping
-      hour = rand(24)
-      minute = rand(6)
-
-      "#{minute}0-#{minute}9 #{hour} * * 0"
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index abef8cd2bcc..c04dae7446f 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -427,10 +427,7 @@ namespace :gitlab do
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
-          File.join(
-            p.repository_storage_path,
-            "#{p.disk_path}.git"
-          )
+          p.repository.path_to_repo
         end
 
         repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index d7f28691098..b854c34a8e5 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -10,9 +10,8 @@ namespace :gitlab do
     end
 
     scope.find_each do |project|
-      base = File.join(project.repository_storage_path, project.disk_path)
-      puts base + '.git'
-      puts base + '.wiki.git'
+      puts project.repository.path_to_repo
+      puts project.wiki.repository.path_to_repo
     end
   end
 end
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index 3cdcdcdf874..c6204f89de4 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -1,7 +1,7 @@
 require Rails.root.join('db/migrate/limits_to_mysql')
 require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
 require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
-require Rails.root.join('db/migrate/limits_ci_job_trace_chunks_raw_data_for_mysql')
+require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
@@ -9,5 +9,5 @@ task add_limits_mysql: :environment do
   LimitsToMysql.new.up
   MarkdownCacheLimitsToMysql.new.up
   MergeRequestDiffFileLimitsToMysql.new.up
-  LimitsCiJobTraceChunksRawDataForMysql.new.up
+  LimitsCiBuildTraceChunksRawDataForMysql.new.up
 end